diff --git a/.gitignore b/.gitignore index 85323f65..163c13e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,35 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +*$py.class # C extensions *.so # Distribution / packaging .Python -env/ -bin/ build/ develop-eggs/ dist/ +downloads/ eggs/ +.eggs/ lib/ lib64/ parts/ sdist/ var/ +wheels/ *.egg-info/ .installed.cfg *.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt @@ -30,28 +39,68 @@ pip-delete-this-directory.txt htmlcov/ .tox/ .coverage +.coverage.* .cache nosetests.xml coverage.xml -cover +*.cover +.hypothesis/ +.pytest_cache/ # Translations *.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject - -# Rope -.ropeproject +*.pot # Django stuff: *.log -*.pot +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy # Sphinx documentation docs/_build/ -# pycharm +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Editors +.vscode/ .idea/ + diff --git a/.pep8speaks.yml b/.pep8speaks.yml new file mode 100644 index 00000000..819887e7 --- /dev/null +++ b/.pep8speaks.yml @@ -0,0 +1,8 @@ +pycodestyle: + max-line-length: 100 # Default is 79 in PEP8 + ignore: # Errors and warnings to ignore + - E401 # multiple imports on one line + exclude: + - "./docs/*" + - "*/build/*" + - "*/migrations/*" diff --git a/.travis.yml b/.travis.yml index dbdbf54c..8670e1e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,83 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +sudo: required +# https://docs.travis-ci.com/user/trusty-ci-environment/ +dist: xenial language: python python: - - "2.6" - - "2.7" -install: "pip install flake8" -script: flake8 --config=flake8.cfg . + # Fedora has 3.6 available (python36.x86_64), but pulp container images + # with it are not being built or published yet. + # - "3.6" + - "3.7" +env: + matrix: + - TEST=pulp + - TEST=docs + - TEST=bindings +matrix: + exclude: + - python: '3.6' + env: TEST=bindings + - python: '3.6' + env: TEST=docs + fast_finish: true +services: + - postgresql + - redis-server + - docker +addons: + apt: + packages: + - httpie + - jq + # postgres versions provided by el7 RHSCL (lowest supportable version) + postgresql: '9.6' +before_install: .travis/before_install.sh +install: .travis/install.sh +before_script: .travis/before_script.sh +script: .travis/script.sh +after_failure: + - http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/ + - sudo docker images + - sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000 + - sudo kubectl logs -l name=pulp-operator -c operator --tail=10000 + - sudo kubectl logs -l app=pulp-api --tail=50000 + - sudo kubectl logs -l app=pulp-content --tail=10000 + - sudo kubectl logs -l app=pulp-resource-manager --tail=10000 + - sudo kubectl logs -l app=pulp-worker --tail=10000 +jobs: + include: + - stage: deploy-plugin-to-pypi + before_install: skip + install: skip + before_script: skip + script: bash .travis/publish_plugin_pypi.sh + if: tag IS present + - stage: publish-daily-client-gem + script: bash .travis/publish_client_gem.sh + env: + - TEST=bindings + if: type = cron + - stage: publish-daily-client-pypi + script: bash .travis/publish_client_pypi.sh + env: + - TEST=bindings + if: type = cron + - stage: publish-client-gem + script: bash .travis/publish_client_gem.sh + env: + - TEST=bindings + if: tag IS present + - stage: publish-client-pypi + script: bash .travis/publish_client_pypi.sh + env: + - TEST=bindings + if: tag IS present +notifications: None + +... diff --git a/.travis/before_install.sh b/.travis/before_install.sh new file mode 100755 index 00000000..d9e9dd85 --- /dev/null +++ b/.travis/before_install.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +export PRE_BEFORE_INSTALL=$TRAVIS_BUILD_DIR/.travis/pre_before_install.sh +export POST_BEFORE_INSTALL=$TRAVIS_BUILD_DIR/.travis/post_before_install.sh + +COMMIT_MSG=$(git log --format=%B --no-merges -1) +export COMMIT_MSG + +if [ -f $PRE_BEFORE_INSTALL ]; then + $PRE_BEFORE_INSTALL +fi + +if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]]; then + echo "Invalid Required PR link detected in commit message. Please use the full https url." + exit 1 +fi + +if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master"] +then + export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}') +else + export PULP_PR_NUMBER= + export PULP_SMASH_PR_NUMBER= + export PULP_ROLES_PR_NUMBER= + export PULP_BINDINGS_PR_NUMBER= + export PULP_OPERATOR_PR_NUMBER= +fi + +# dev_requirements contains tools needed for flake8, etc. +# So install them here rather than in install.sh +pip install -r dev_requirements.txt + +# check the commit message +./.travis/check_commit.sh + + + +# Lint code. +flake8 --config flake8.cfg + +cd .. +git clone --depth=1 https://github.com/pulp/ansible-pulp.git +if [ -n "$PULP_ROLES_PR_NUMBER" ]; then + cd ansible-pulp + git fetch --depth=1 origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge + git checkout FETCH_HEAD + cd .. +fi + + +git clone --depth=1 https://github.com/pulp/pulp-operator.git +if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then + cd pulp-operator + git fetch --depth=1 origin +refs/pull/$PULP_OPERATOR_PR_NUMBER/merge + git checkout FETCH_HEAD + RELEASE_VERSION=v0.9.0 + curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu + chmod +x operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu && sudo mkdir -p /usr/local/bin/ && sudo cp operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu + sudo operator-sdk build --image-builder=docker quay.io/pulp/pulp-operator:latest + cd .. +fi + + +git clone --depth=1 https://github.com/pulp/pulpcore.git + +if [ -n "$PULP_PR_NUMBER" ]; then + cd pulpcore + git fetch --depth=1 origin +refs/pull/$PULP_PR_NUMBER/merge + git checkout FETCH_HEAD + cd .. +fi + + + +# When building a (release) tag, we don't need the development modules for the +# build (they will be installed as dependencies of the plugin). +if [ -z "$TRAVIS_TAG" ]; then + + git clone --depth=1 https://github.com/PulpQE/pulp-smash.git + + if [ -n "$PULP_SMASH_PR_NUMBER" ]; then + cd pulp-smash + git fetch --depth=1 origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge + git checkout FETCH_HEAD + cd .. + fi + + # pulp-smash already got installed via test_requirements.txt + pip install --upgrade --force-reinstall ./pulp-smash + +fi + + +pip install ansible + +cd pulp_docker + +if [ -f $POST_BEFORE_INSTALL ]; then + $POST_BEFORE_INSTALL +fi diff --git a/.travis/before_script.sh b/.travis/before_script.sh new file mode 100755 index 00000000..b4938900 --- /dev/null +++ b/.travis/before_script.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +export PRE_BEFORE_SCRIPT=$TRAVIS_BUILD_DIR/.travis/pre_before_script.sh +export POST_BEFORE_SCRIPT=$TRAVIS_BUILD_DIR/.travis/post_before_script.sh + +# Aliases for running commands in the pulp-api container. +export PULP_API_POD=$(sudo kubectl get pods | grep -E -o "pulp-api-(\w+)-(\w+)") +# Run a command +export CMD_PREFIX="sudo kubectl exec $PULP_API_POD --" +# Run a command, and pass STDIN +export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --" + +if [[ -f $PRE_BEFORE_SCRIPT ]]; then + $PRE_BEFORE_SCRIPT +fi + +mkdir -p ~/.config/pulp_smash + +if [[ -f .travis/pulp-smash-config.json ]]; then + sed "s/localhost/$(hostname)/g" .travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json +else + sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json +fi + +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then + # Many tests require pytest/mock, but users do not need them at runtime + # (or to add plugins on top of pulpcore or pulp container images.) + # So install it here, rather than in the image Dockerfile. + $CMD_PREFIX pip3 install pytest mock + # Many functional tests require these + $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core +fi + +if [[ -f $POST_BEFORE_SCRIPT ]]; then + $POST_BEFORE_SCRIPT +fi diff --git a/.travis/check_commit.sh b/.travis/check_commit.sh new file mode 100755 index 00000000..cda7e60f --- /dev/null +++ b/.travis/check_commit.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +# skip this check for everything but PRs +if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then + exit 0 +fi + +if [ "$TRAVIS_COMMIT_RANGE" != "" ]; then + RANGE=$TRAVIS_COMMIT_RANGE +elif [ "$TRAVIS_COMMIT" != "" ]; then + RANGE=$TRAVIS_COMMIT +fi + +# Travis sends the ranges with 3 dots. Git only wants two. +if [[ "$RANGE" == *...* ]]; then + RANGE=`echo $TRAVIS_COMMIT_RANGE | sed 's/\.\.\./../'` +else + RANGE="$RANGE~..$RANGE" +fi + +for sha in `git log --format=oneline --no-merges "$RANGE" | cut '-d ' -f1` +do + python .travis/validate_commit_message.py $sha + VALUE=$? + + if [ "$VALUE" -gt 0 ]; then + exit $VALUE + fi +done diff --git a/.travis/install.sh b/.travis/install.sh new file mode 100755 index 00000000..c25f1124 --- /dev/null +++ b/.travis/install.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +if [ "$TEST" = 'docs' ]; then + + pip install -r ../pulpcore/doc_requirements.txt + + pip install -r doc_requirements.txt +fi + +pip install -r functest_requirements.txt + +cd $TRAVIS_BUILD_DIR/../pulpcore/containers/ + +# Although the tag name is not used outside of this script, we might use it +# later. And it is nice to have a friendly identifier for it. +# So we use the branch preferably, but need to replace the "/" with the valid +# character "_" . +# +# Note that there are lots of other valid git branch name special characters +# that are invalid in image tag names. To try to convert them, this would be a +# starting point: +# https://stackoverflow.com/a/50687120 +# +# If we are on a tag +if [ -n "$TRAVIS_TAG" ]; then + TAG=$(echo $TRAVIS_TAG | tr / _) +# If we are on a PR +elif [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]; then + TAG=$(echo $TRAVIS_PULL_REQUEST_BRANCH | tr / _) +# For push builds and hopefully cron builds +elif [ -n "$TRAVIS_BRANCH" ]; then + TAG=$(echo $TRAVIS_BRANCH | tr / _) + if [ "$TAG" = "master" ]; then + TAG=latest + fi +else + # Fallback + TAG=$(git rev-parse --abbrev-ref HEAD | tr / _) +fi + +PLUGIN=pulp_docker + + +# For pulpcore, and any other repo that might check out some plugin PR +if [ -n "$TRAVIS_TAG" ]; then + # Install the plugin only and use published PyPI packages for the rest + cat > vars/vars.yaml << VARSYAML +--- +images: + - ${PLUGIN}-${TAG}: + image_name: $PLUGIN + tag: $TAG + plugins: + - ./$PLUGIN +VARSYAML +else + cat > vars/vars.yaml << VARSYAML +--- +images: + - ${PLUGIN}-${TAG}: + image_name: $PLUGIN + tag: $TAG + pulpcore: ./pulpcore + plugins: + - ./$PLUGIN +VARSYAML +fi +ansible-playbook build.yaml + +cd $TRAVIS_BUILD_DIR/../pulp-operator +# Tell pulp-perator to deploy our image +cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML +apiVersion: pulpproject.org/v1alpha1 +kind: Pulp +metadata: + name: example-pulp +spec: + pulp_file_storage: + # k3s local-path requires this + access_mode: "ReadWriteOnce" + # We have a little over 40GB free on Travis VMs/instances + size: "40Gi" + image: $PLUGIN + tag: $TAG + database_connection: + username: pulp + password: pulp + admin_password: pulp + pulp_settings: + token_server: $(hostname):24816/token + private_key_path: /var/lib/pulp/tmp/private.pem + public_key_path: /var/lib/pulp/tmp/public.pem + token_signature_algorithm: ES256 +CRYAML + +# Install k3s, lightweight Kubernetes +.travis/k3s-install.sh +# Deploy pulp-operator, with the pulp containers, according to CRYAML +sudo ./up.sh + +# Needed for the script below +# Since it is being run during install rather than actual tests (unlike in +# pulp-operator), and therefore does not trigger the equivalent after_failure +# travis commands. +show_logs_and_return_non_zero() { + readonly local rc="$?" + + for containerlog in "pulp-api" "pulp-content" "pulp-resource-manager" "pulp-worker" + do + echo -en "travis_fold:start:$containerlog"'\\r' + sudo kubectl logs -l app=$containerlog --tail=10000 + echo -en "travis_fold:end:$containerlog"'\\r' + done + + return "${rc}" +} +.travis/pulp-operator-check-and-wait.sh || show_logs_and_return_non_zero diff --git a/.travis/mariadb.yml b/.travis/mariadb.yml new file mode 100644 index 00000000..b26bda83 --- /dev/null +++ b/.travis/mariadb.yml @@ -0,0 +1,9 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +pulp_db_backend: django.db.backends.mysql diff --git a/.travis/post_before_script.sh b/.travis/post_before_script.sh new file mode 100755 index 00000000..ad55ad65 --- /dev/null +++ b/.travis/post_before_script.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +echo "machine localhost +login admin +password password + +machine 127.0.0.1 +login admin +password password +" > ~/.netrc + +$CMD_PREFIX bash -c "dnf install -y openssl" +$CMD_PREFIX bash -c "openssl ecparam -genkey -name prime256v1 -noout -out /var/lib/pulp/tmp/private.key" +$CMD_PREFIX bash -c "openssl ec -in /var/lib/pulp/tmp/private.key -pubout -out /var/lib/pulp/tmp/public.key" diff --git a/.travis/post_docs_test.sh b/.travis/post_docs_test.sh new file mode 100755 index 00000000..a306c539 --- /dev/null +++ b/.travis/post_docs_test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +cd docs/_scripts/ +bash ./docs_check.sh diff --git a/.travis/postgres.yml b/.travis/postgres.yml new file mode 100644 index 00000000..295bd570 --- /dev/null +++ b/.travis/postgres.yml @@ -0,0 +1,9 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +pulp_db_backend: django.db.backends.postgresql_psycopg2 diff --git a/.travis/publish_client_gem.sh b/.travis/publish_client_gem.sh new file mode 100755 index 00000000..93b41ac5 --- /dev/null +++ b/.travis/publish_client_gem.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +echo "--- +:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +sudo chmod 600 ~/.gem/credentials + +django-admin runserver 24817 >> ~/django_runserver.log 2>&1 & +sleep 5 + +cd $TRAVIS_BUILD_DIR +export REPORTED_VERSION=$(http :24817/pulp/api/v3/status/ | jq --arg plugin pulp_docker -r '.versions[] | select(.component == $plugin) | .version') +export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" +if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then + export VERSION=${REPORTED_VERSION} +else + # Daily publishing of development version (ends in ".dev" reported as ".dev0") + [ "${REPORTED_VERSION%.dev*}" != "${REPORTED_VERSION}" ] || exit 1 + export EPOCH="$(date +%s)" + export VERSION=${REPORTED_VERSION}${EPOCH} +fi + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_docker_client/versions/$VERSION) + +if [ "$response" == "200" ]; +then + exit +fi + +cd +git clone https://github.com/pulp/pulp-openapi-generator.git +cd pulp-openapi-generator + +./generate.sh pulp_docker ruby $VERSION +cd pulp_docker-client +gem build pulp_docker_client +GEM_FILE="$(ls | grep pulp_docker_client-)" +gem push ${GEM_FILE} diff --git a/.travis/publish_client_pypi.sh b/.travis/publish_client_pypi.sh new file mode 100755 index 00000000..f0cd37f9 --- /dev/null +++ b/.travis/publish_client_pypi.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +pip install twine + +django-admin runserver 24817 >> ~/django_runserver.log 2>&1 & +sleep 5 + +cd "${TRAVIS_BUILD_DIR}" +export REPORTED_VERSION=$(http :24817/pulp/api/v3/status/ | jq --arg plugin pulp_docker -r '.versions[] | select(.component == $plugin) | .version') +export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" +if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then + export VERSION=${REPORTED_VERSION} +else + # Daily publishing of development version (ends in ".dev" reported as ".dev0") + [ "${REPORTED_VERSION%.dev*}" != "${REPORTED_VERSION}" ] || exit 1 + export EPOCH="$(date +%s)" + export VERSION=${REPORTED_VERSION}${EPOCH} +fi + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-docker-client/$VERSION/) + +if [ "$response" == "200" ]; +then + exit +fi + +cd +git clone https://github.com/pulp/pulp-openapi-generator.git +cd pulp-openapi-generator + +./generate.sh pulp_docker python $VERSION +cd pulp_docker-client +python setup.py sdist bdist_wheel --python-tag py3 +twine upload dist/* -u pulp -p $PYPI_PASSWORD +exit $? diff --git a/.travis/publish_plugin_pypi.sh b/.travis/publish_plugin_pypi.sh new file mode 100755 index 00000000..1c603dfe --- /dev/null +++ b/.travis/publish_plugin_pypi.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +pip install twine + +python setup.py sdist bdist_wheel --python-tag py3 +twine upload dist/* -u pulp -p $PYPI_PASSWORD diff --git a/.travis/pulp-smash-config.json b/.travis/pulp-smash-config.json new file mode 100644 index 00000000..fcaede33 --- /dev/null +++ b/.travis/pulp-smash-config.json @@ -0,0 +1,22 @@ + +{ + "pulp": { + "auth": ["admin", "password"], + "selinux enabled": false, + "version": "3" + }, + "hosts": [ + { + "hostname": "localhost", + "roles": { + "api": {"port": 24817, "scheme": "http", "service": "nginx"}, + "content": {"port": 24816, "scheme": "http", "service": "pulp_content_app"}, + "token auth": {"private key": "/var/lib/pulp/tmp/private.pem", "public key": "/var/lib/pulp/tmp/public.pem"}, + "pulp resource manager": {}, + "pulp workers": {}, + "redis": {}, + "shell": {"transport": "kubectl"} + } + } + ] +} diff --git a/.travis/script.sh b/.travis/script.sh new file mode 100755 index 00000000..c8d68786 --- /dev/null +++ b/.travis/script.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# coding=utf-8 + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +export POST_SCRIPT=$TRAVIS_BUILD_DIR/.travis/post_script.sh +export POST_DOCS_TEST=$TRAVIS_BUILD_DIR/.travis/post_docs_test.sh +export FUNC_TEST_SCRIPT=$TRAVIS_BUILD_DIR/.travis/func_test_script.sh + +# Needed for both starting the service and building the docs. +# Gets set in .travis/settings.yml, but doesn't seem to inherited by +# this script. +export DJANGO_SETTINGS_MODULE=pulpcore.app.settings + +if [ "$TEST" = 'docs' ]; then + cd docs + make html + cd .. + + if [ -f $POST_DOCS_TEST ]; then + $POST_DOCS_TEST + fi + exit +fi + +if [ "$TEST" = 'bindings' ]; then + COMMIT_MSG=$(git log --format=%B --no-merges -1) + export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') + + cd .. + git clone https://github.com/pulp/pulp-openapi-generator.git + cd pulp-openapi-generator + + if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then + git fetch origin +refs/pull/$PULP_BINDINGS_PR_NUMBER/merge + git checkout FETCH_HEAD + fi + + ./generate.sh pulpcore python + pip install ./pulpcore-client + ./generate.sh pulp_docker python + pip install ./pulp_docker-client + + python $TRAVIS_BUILD_DIR/.travis/test_bindings.py + + if [ ! -f $TRAVIS_BUILD_DIR/.travis/test_bindings.rb ] + then + exit + fi + + rm -rf ./pulpcore-client + + ./generate.sh pulpcore ruby + cd pulpcore-client + gem build pulpcore_client + gem install --both ./pulpcore_client-0.gem + cd .. + + rm -rf ./pulp_docker-client + + ./generate.sh pulp_docker ruby + + cd pulp_docker-client + gem build pulp_docker_client + gem install --both ./pulp_docker_client-0.gem + cd .. + + ruby $TRAVIS_BUILD_DIR/.travis/test_bindings.rb + exit +fi + +# Aliases for running commands in the pulp-api container. +export PULP_API_POD=$(sudo kubectl get pods | grep -E -o "pulp-api-(\w+)-(\w+)") +# Run a command +export CMD_PREFIX="sudo kubectl exec $PULP_API_POD --" +# Run a command, and pass STDIN +export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --" +# The alias does not seem to work in Travis / the scripting framework +#alias pytest="$CMD_PREFIX pytest" + +cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt" +$CMD_PREFIX pip3 install -r /tmp/test_requirements.txt + +# Run unit tests. +$CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulp_docker/tests/unit/" + +# Note: This function is in the process of being merged into after_failure +show_logs_and_return_non_zero() { + readonly local rc="$?" + return "${rc}" +} +export -f show_logs_and_return_non_zero + +# Run functional tests +set +u + +export PYTHONPATH=$TRAVIS_BUILD_DIR:$TRAVIS_BUILD_DIR/../pulpcore:${PYTHONPATH} + +set -u + +if [[ "$TEST" == "performance" ]]; then + echo "--- Performance Tests ---" + pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_docker.tests.performance || show_logs_and_return_non_zero + exit +fi + +if [ -f $FUNC_TEST_SCRIPT ]; then + $FUNC_TEST_SCRIPT +else + pytest -v -r sx --color=yes --pyargs pulp_docker.tests.functional || show_logs_and_return_non_zero +fi + +if [ -f $POST_SCRIPT ]; then + $POST_SCRIPT +fi diff --git a/common/pulp_docker/common/__init__.py b/.travis/test_bindings.py similarity index 100% rename from common/pulp_docker/common/__init__.py rename to .travis/test_bindings.py diff --git a/.travis/validate_commit_message.py b/.travis/validate_commit_message.py new file mode 100644 index 00000000..a949d314 --- /dev/null +++ b/.travis/validate_commit_message.py @@ -0,0 +1,60 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by bootstrap.py. Please use +# bootstrap.py to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import glob +import re +import requests +import subprocess +import sys + +KEYWORDS = ["fixes", "closes", "re", "ref"] +NO_ISSUE = "[noissue]" +STATUSES = ["NEW", "ASSIGNED", "POST"] + +sha = sys.argv[1] +message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + +def __check_status(issue): + response = requests.get("https://pulp.plan.io/issues/{}.json".format(issue)) + response.raise_for_status() + bug_json = response.json() + status = bug_json["issue"]["status"]["name"] + if status not in STATUSES: + sys.exit( + "Error: issue #{issue} has invalid status of {status}. Status must be one of " + "{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES)) + ) + + +def __check_changelog(issue): + if len(glob.glob(f"CHANGES/**/{issue}.*", recursive=True)) < 1: + sys.exit(f"Could not find changelog entry in CHANGES/ for {issue}.") + + +print("Checking commit message for {sha}.".format(sha=sha[0:7])) + +# validate the issue attached to the commit +if NO_ISSUE in message: + print("Commit {sha} has no issue attached. Skipping issue check".format(sha=sha[0:7])) +else: + regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) + pattern = re.compile(regex) + + issues = pattern.findall(message) + + if issues: + for issue in pattern.findall(message): + __check_status(issue) + __check_changelog(issue) + else: + sys.exit( + "Error: no attached issues found for {sha}. If this was intentional, add " + " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) + ) + +print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/AUTHORS b/AUTHORS index bcb2d392..74179905 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,11 @@ -Barnaby Court (bcourt@redhat.com) -Michael Hrivnak (mhrivnak@redhat.com) -Randy Barlow (rbarlow@redhat.com) -Sayli Karmarkar (skarmark@redhat.com) +Muhammad Ammar Ansari +Randy Barlow +Barnaby Court +Patrick Creech +Michael Hrivnak +Sayli Karmarkar +Jindrich Luza +Austin Macdonald +Rohan McGovern +Tomáš Tomeček +Mihai Ibanescu diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 00000000..4d5a15e4 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,115 @@ +========= +Changelog +========= + +.. + You should *NOT* be adding new change log entries to this file, this + file is managed by towncrier. You *may* edit previous change logs to + fix problems like typo corrections or such. + To add a new change log entry, please see + https://docs.pulpproject.org/en/3.0/nightly/contributing/git.html#changelog-update + + WARNING: Don't drop the next directive! + +.. towncrier release notes start + +4.0.0b7 (2019-10-02) +==================== + + +Bugfixes +-------- + +- Fix a bug that allowed arbitrary url prefixes for custom endpoints. + `#5486 `_ +- Add Docker-Distribution-API-Version header among response headers. + `#5527 `_ + + +Misc +---- + +- `#5470 `_ + + +---- + + +4.0.0b6 (2019-09-05) +==================== + + +Features +-------- + +- Add endpoint to recursively copy manifests from a source repository to a destination repository. + `#3403 `_ +- Add endpoint to recursively add docker content to a repository. + `#3405 `_ +- As a user I can sync from a docker repo published by Pulp2/Pulp3. + `#4737 `_ +- Add support for tagging and untagging manifests via an additional endpoint + `#4934 `_ +- Add endpoint for copying all tags from a source repository, or specific tags by name. + `#4947 `_ +- Add ability to filter Manifests and ManifestTags by media_type and digest + `#5033 `_ +- Add ability to filter Manifests, ManifestTags and Blobs by multiple media_types + `#5157 `_ +- Add endpoint to recursively remove docker content from a repository. + `#5179 `_ + + +Bugfixes +-------- + +- Allow Accept header to send multiple values. + `#5211 `_ +- Populate ManifestListManifest thru table during sync. + `#5235 `_ +- Fixed a problem where repeated syncs created invalid orphaned tags. + `#5252 `_ + + +Misc +---- + +- `#4681 `_, `#5213 `_, `#5218 `_ + + +---- + + +4.0.0b5 (2019-07-04) +==================== + + +Bugfixes +-------- + +- Add 'Docker-Content-Digest' header to the response headers. + `#4646 `_ +- Allow docker remote whitelist_tags to be unset to null. + `#5017 `_ +- Remove schema1 manifest signature when calculating its digest. + `#5037 `_ + + +Improved Documentation +---------------------- + +- Switch to using `towncrier `_ for better release notes. + `#4875 `_ +- Add an example to the whitelist_tag help text + `#4994 `_ +- Add list of features to the docker landing page. + `#5030 `_ + + +Misc +---- + +- `#4572 `_, `#4994 `_, `#5014 `_ + + +---- diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.rst new file mode 100644 index 00000000..ab3826e7 --- /dev/null +++ b/CHANGES/.TEMPLATE.rst @@ -0,0 +1,37 @@ + +{# TOWNCRIER TEMPLATE #} +{% for section, _ in sections.items() %} +{% set underline = underlines[0] %}{% if section %}{{section}} +{{ underline * section|length }}{% set underline = underlines[1] %} + +{% endif %} + +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section]%} +{{ definitions[category]['name'] }} +{{ underline * definitions[category]['name']|length }} + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category].items() %} +- {{ text }} + {{ values|join(',\n ') }} +{% endfor %} + +{% else %} +- {{ sections[section][category]['']|join(', ') }} + +{% endif %} +{% if sections[section][category]|length == 0 %} +No significant changes. + +{% else %} +{% endif %} + +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} +{% endfor %} +---- diff --git a/CHANGES/.gitignore b/CHANGES/.gitignore new file mode 100644 index 00000000..f935021a --- /dev/null +++ b/CHANGES/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/CHANGES/4244.feature b/CHANGES/4244.feature new file mode 100644 index 00000000..4aab514f --- /dev/null +++ b/CHANGES/4244.feature @@ -0,0 +1 @@ +Convert manifests of the format schema 2 to schema 1 diff --git a/CHANGES/4554.doc b/CHANGES/4554.doc new file mode 100644 index 00000000..7209b71b --- /dev/null +++ b/CHANGES/4554.doc @@ -0,0 +1 @@ +Change the prefix of Pulp services from pulp-* to pulpcore-* diff --git a/CHANGES/4938.feature b/CHANGES/4938.feature new file mode 100644 index 00000000..f5af47f2 --- /dev/null +++ b/CHANGES/4938.feature @@ -0,0 +1 @@ +Add support for pulling content using token authentication \ No newline at end of file diff --git a/CHANGES/5454.removal b/CHANGES/5454.removal new file mode 100644 index 00000000..24c4e615 --- /dev/null +++ b/CHANGES/5454.removal @@ -0,0 +1 @@ +Change `_type` to `pulp_type` diff --git a/CHANGES/5457.removal b/CHANGES/5457.removal new file mode 100644 index 00000000..5eb02968 --- /dev/null +++ b/CHANGES/5457.removal @@ -0,0 +1 @@ +Change `_id`, `_created`, `_last_updated`, `_href` to `pulp_id`, `pulp_created`, `pulp_last_updated`, `pulp_href` diff --git a/CHANGES/5515.feature b/CHANGES/5515.feature new file mode 100644 index 00000000..823be676 --- /dev/null +++ b/CHANGES/5515.feature @@ -0,0 +1 @@ +Store whitelisted tags in a list instead of CSV string diff --git a/CHANGES/5548.removal b/CHANGES/5548.removal new file mode 100644 index 00000000..be1e5672 --- /dev/null +++ b/CHANGES/5548.removal @@ -0,0 +1 @@ +Remove "_" from `_versions_href`, `_latest_version_href` diff --git a/CHANGES/5550.removal b/CHANGES/5550.removal new file mode 100644 index 00000000..d75a8cc8 --- /dev/null +++ b/CHANGES/5550.removal @@ -0,0 +1 @@ +Removing base field: `_type` . diff --git a/CHANGES/5580.misc b/CHANGES/5580.misc new file mode 100644 index 00000000..c53f440c --- /dev/null +++ b/CHANGES/5580.misc @@ -0,0 +1,2 @@ +Depend on pulpcore, directly, instead of pulpcore-plugin. + diff --git a/CHANGES/5635.feature b/CHANGES/5635.feature new file mode 100644 index 00000000..5d35c237 --- /dev/null +++ b/CHANGES/5635.feature @@ -0,0 +1 @@ +Added v2s2 to v2s1 converter. diff --git a/CHANGES/5637.bugfix b/CHANGES/5637.bugfix new file mode 100644 index 00000000..445c1e4d --- /dev/null +++ b/CHANGES/5637.bugfix @@ -0,0 +1 @@ +Fix using specified proxy for downloads. diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..893a497b --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,2 @@ +# have QE own the functional tests +/pulp_docker/tests/functional/ @pulp/qe diff --git a/COMMITMENT b/COMMITMENT new file mode 100644 index 00000000..99a5cf07 --- /dev/null +++ b/COMMITMENT @@ -0,0 +1,45 @@ +GPL Cooperation Commitment, version 1.0 + +Before filing or continuing to prosecute any legal proceeding or claim +(other than a Defensive Action) arising from termination of a Covered +License, we commit to extend to the person or entity ('you') accused +of violating the Covered License the following provisions regarding +cure and reinstatement, taken from GPL version 3. As used here, the +term 'this License' refers to the specific Covered License being +enforced. + + However, if you cease all violation of this License, then your + license from a particular copyright holder is reinstated (a) + provisionally, unless and until the copyright holder explicitly + and finally terminates your license, and (b) permanently, if the + copyright holder fails to notify you of the violation by some + reasonable means prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is + reinstated permanently if the copyright holder notifies you of the + violation by some reasonable means, this is the first time you + have received notice of violation of this License (for any work) + from that copyright holder, and you cure the violation prior to 30 + days after your receipt of the notice. + +We intend this Commitment to be irrevocable, and binding and +enforceable against us and assignees of or successors to our +copyrights. + +Definitions + +'Covered License' means the GNU General Public License, version 2 +(GPLv2), the GNU Lesser General Public License, version 2.1 +(LGPLv2.1), or the GNU Library General Public License, version 2 +(LGPLv2), all as published by the Free Software Foundation. + +'Defensive Action' means a legal proceeding or claim that We bring +against you in response to a prior proceeding or claim initiated by +you or your affiliate. + +'We' means each contributor to this repository as of the date of +inclusion of this file, including subsidiaries of a corporate +contributor. + +This work is available under a [Creative Commons Attribution-ShareAlike 4.0 International license] +(https://creativecommons.org/licenses/by-sa/4.0/). diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..73dc3d12 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,35 @@ +Contributing +============ + +To contribute to the ``pulp_docker`` package follow this process: + +1. Clone the GitHub repo +2. Make a change +3. Make sure all tests passed +4. Add a file into CHANGES folder (Changelog update). +5. Commit changes to own ``pulp_docker`` clone +6. Make pull request from github page for your clone against master branch + + +.. _changelog-update: + +Changelog update +**************** + +The CHANGES.rst file is managed using the `towncrier tool `_ +and all non trivial changes must be accompanied by a news entry. + +To add an entry to the news file, you first need an issue in pulp.plan.io describing the change you +want to make. Once you have an issue, take its number and create a file inside of the ``CHANGES/`` +directory named after that issue number with an extension of .feature, .bugfix, .doc, .removal, or +.misc. So if your issue is 3543 and it fixes a bug, you would create the file +``CHANGES/3543.bugfix``. + +PRs can span multiple categories by creating multiple files (for instance, if you added a feature +and deprecated an old feature at the same time, you would create CHANGES/NNNN.feature and +CHANGES/NNNN.removal). Likewise if a PR touches multiple issues/PRs you may create a file for each +of them with the exact same contents and Towncrier will deduplicate them. + +The contents of this file are reStructuredText formatted text that will be used as the content of +the news file entry. You do not need to reference the issue or PR numbers here as towncrier will +automatically add a reference to all of the affected issues when rendering the news file. diff --git a/COPYRIGHT b/COPYRIGHT index c39fad67..22fbe5db 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,10 +1,339 @@ -Copyright © 2014 Pulp Project developers. - -This software is licensed to you under the GNU General Public -License as published by the Free Software Foundation; either version -2 of the License (GPLv2) or (at your option) any later version. -There is NO WARRANTY for this software, express or implied, -including the implied warranties of MERCHANTABILITY, -NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -have received a copy of GPLv2 along with this software; if not, see -http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {description} + Copyright (C) {year} {fullname} + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + {signature of Ty Coon}, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 00000000..d91fc9db --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,35 @@ +4.0.0b4 +^^^^^^^ + +- Enable sync from registries that use basic auth or have private repos +- Enable on_demand or streamed sync +- Enable whitelist tags specification when syncing +- Compatibility with pulpcore-plugin-0.1.0rc2 + +`Comprehensive list of changes and bugfixes for beta 4 `_. + +4.0.0b3 +^^^^^^^ + +- Enable sync from gcr and quay registries +- Enable support to handle pagination for tags/list endpoint during sync +- Enable support to manage a docker image that has manifest schema v2s1 +- Enable docker distribution to serve directly latest repository version + +`Comprehensive list of changes and bugfixes for beta 3 `_. + +4.0.0b2 +^^^^^^^ + +- Compatibility with pulpcore-plugin-0.1.0rc1 +- Performance improvements and bug fixes +- Add support for syncing repo with foreign layers +- Change sync pipeline to use Futures to handle nested content +- Make Docker distributions asyncronous +- Add support to create publication directly + +4.0.0b1 +^^^^^^^ + +- Add support for basic sync of a docker repo form a V2Registry +- Add support for docker/podman pull from a docker distbution served by Pulp diff --git a/LICENSE b/LICENSE index 22fbe5db..23cb7903 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -GNU GENERAL PUBLIC LICENSE + GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., @@ -336,4 +336,4 @@ This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. \ No newline at end of file +Public License instead of this License. diff --git a/README.md b/README.md deleted file mode 100644 index 8580e3ab..00000000 --- a/README.md +++ /dev/null @@ -1,9 +0,0 @@ -pulp_docker -=========== - -tagging -------- - -To tag a new version, edit pulp-docker.spec to the new version, create a PR, -and once merged run `tito tag --keep-version`. Do not use the tito -auto-increment tagger for the time being. diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..cc31e34b --- /dev/null +++ b/README.rst @@ -0,0 +1,13 @@ +``pulp_docker`` Plugin +====================== + +.. image:: https://travis-ci.org/pulp/pulp_docker.svg?branch=master + :target: https://travis-ci.org/pulp/pulp_docker + +This is the ``pulp_docker`` Plugin for `Pulp Project +3.0+ `__. This plugin provides Pulp with support for docker +images, similar to the pulp_docker plugin for Pulp 2. + +For more information, please see the `documentation +`_ or the `Pulp project page +`_. diff --git a/common/pulp_docker/__init__.py b/common/pulp_docker/__init__.py deleted file mode 100644 index 3ad9513f..00000000 --- a/common/pulp_docker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/common/pulp_docker/common/constants.py b/common/pulp_docker/common/constants.py deleted file mode 100644 index 1084eb92..00000000 --- a/common/pulp_docker/common/constants.py +++ /dev/null @@ -1,46 +0,0 @@ -IMAGE_TYPE_ID = 'docker_image' -IMPORTER_TYPE_ID = 'docker_importer' -IMPORTER_CONFIG_FILE_NAME = 'server/plugins.conf.d/docker_importer.json' -DISTRIBUTOR_WEB_TYPE_ID = 'docker_distributor_web' -DISTRIBUTOR_EXPORT_TYPE_ID = 'docker_distributor_export' -CLI_WEB_DISTRIBUTOR_ID = 'docker_web_distributor_name_cli' -CLI_EXPORT_DISTRIBUTOR_ID = 'docker_export_distributor_name_cli' -DISTRIBUTOR_CONFIG_FILE_NAME = 'server/plugins.conf.d/docker_distributor.json' -DISTRIBUTOR_EXPORT_CONFIG_FILE_NAME = 'server/plugins.conf.d/docker_distributor_export.json' - -REPO_NOTE_DOCKER = 'docker-repo' - -# Config keys for the importer -CONFIG_KEY_UPSTREAM_NAME = 'upstream_name' - -# Config keys for the distributor plugin conf -CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY = 'docker_publish_directory' -CONFIG_VALUE_DOCKER_PUBLISH_DIRECTORY = '/var/lib/pulp/published/docker' -CONFIG_KEY_EXPORT_FILE = 'export_file' - -# Config keys for a distributor instance in the database -CONFIG_KEY_REDIRECT_URL = 'redirect-url' -CONFIG_KEY_PROTECTED = 'protected' -CONFIG_KEY_REPO_REGISTRY_ID = 'repo-registry-id' - -# Config keys for an importer override config -CONFIG_KEY_MASK_ID = 'mask_id' - -SYNC_STEP_MAIN = 'sync_step_main' -SYNC_STEP_METADATA = 'sync_step_metadata' -SYNC_STEP_GET_LOCAL = 'sync_step_metadata' -SYNC_STEP_DOWNLOAD = 'sync_step_download' -SYNC_STEP_SAVE = 'sync_step_save' - -# Keys that are specified on the repo config -PUBLISH_STEP_WEB_PUBLISHER = 'publish_to_web' -PUBLISH_STEP_EXPORT_PUBLISHER = 'export_to_tar' -PUBLISH_STEP_IMAGES = 'publish_images' -PUBLISH_STEP_OVER_HTTP = 'publish_images_over_http' -PUBLISH_STEP_DIRECTORY = 'publish_directory' -PUBLISH_STEP_TAR = 'save_tar' - -# Dictionary keys to be used when storing or accessing a list of tag dictionaries -# on the repo scratchpad -IMAGE_TAG_KEY = 'tag' -IMAGE_ID_KEY = 'image_id' diff --git a/common/pulp_docker/common/error_codes.py b/common/pulp_docker/common/error_codes.py deleted file mode 100644 index c5cd8110..00000000 --- a/common/pulp_docker/common/error_codes.py +++ /dev/null @@ -1,12 +0,0 @@ -from gettext import gettext as _ - -from pulp.common.error_codes import Error - -DKR1001 = Error("DKR1001", _("The url specified for %(field) is missing a scheme. " - "The value specified is '%(url)'."), ['field', 'url']) -DKR1002 = Error("DKR1002", _("The url specified for %(field) is missing a hostname. " - "The value specified is '%(url)'."), ['field', 'url']) -DKR1003 = Error("DKR1003", _("The url specified for %(field) is missing a path. " - "The value specified is '%(url)'."), ['field', 'url']) -DKR1004 = Error("DKR1004", _("The value specified for %(field): '%(value)s' is not boolean."), - ['field', 'value']) diff --git a/common/pulp_docker/common/models.py b/common/pulp_docker/common/models.py deleted file mode 100644 index 8eef23e8..00000000 --- a/common/pulp_docker/common/models.py +++ /dev/null @@ -1,52 +0,0 @@ -import os - -from pulp_docker.common import constants - - -class DockerImage(object): - TYPE_ID = constants.IMAGE_TYPE_ID - - def __init__(self, image_id, parent_id, size): - """ - :param image_id: unique image ID - :type image_id: basestring - :param parent_id: parent's unique image ID - :type parent_id: basestring - :param size: size of the image in bytes, as reported by docker. - This can be None, because some very old docker images - do not contain it in their metadata. - :type size: int or NoneType - """ - self.image_id = image_id - self.parent_id = parent_id - self.size = size - - @property - def unit_key(self): - """ - :return: unit key - :rtype: dict - """ - return { - 'image_id': self.image_id - } - - @property - def relative_path(self): - """ - :return: the relative path to where this image's directory should live - :rtype: basestring - """ - return os.path.join(self.TYPE_ID, self.image_id) - - @property - def unit_metadata(self): - """ - :return: a subset of the complete docker metadata about this image, - including only what pulp_docker cares about - :rtype: dict - """ - return { - 'parent_id': self.parent_id, - 'size': self.size - } diff --git a/common/pulp_docker/common/tags.py b/common/pulp_docker/common/tags.py deleted file mode 100644 index ecd2a561..00000000 --- a/common/pulp_docker/common/tags.py +++ /dev/null @@ -1,28 +0,0 @@ -from pulp_docker.common import constants - - -def generate_updated_tags(scratchpad, new_tags): - """ - Get the current repo scratchpad's tags and generate an updated tag list - by adding new tags to them. If a tag exists on the scratchpad as well as - in the new tags, the old tag will be overwritten by the new tag. - - :param scratchpad: repo scratchpad dictionary - :type scratchpad: dict - :param new_tags: dictionary of tag:image_id - :type new_tags: dict - :return: list of dictionaries each containing values for 'tag' and 'image_id' keys - :rtype: list of dict - """ - tags = scratchpad.get('tags', []) - - # Remove common tags between existing and new tags so we don't have duplicates - for tag_dict in tags[:]: - if tag_dict[constants.IMAGE_TAG_KEY] in new_tags.keys(): - tags.remove(tag_dict) - # Add new tags to existing tags. Since tags can contain '.' which cannot be stored - # as a key in mongodb, we are storing them this way. - for tag, image_id in new_tags.items(): - tags.append({constants.IMAGE_TAG_KEY: tag, constants.IMAGE_ID_KEY: image_id}) - - return tags diff --git a/common/pulp_docker/common/tarutils.py b/common/pulp_docker/common/tarutils.py deleted file mode 100644 index 86267933..00000000 --- a/common/pulp_docker/common/tarutils.py +++ /dev/null @@ -1,116 +0,0 @@ -import contextlib -import json -import os -import tarfile - - -def get_metadata(tarfile_path): - """ - Given a path to a tarfile, which is itself the product of "docker save", - this discovers what images (layers) exist in the archive and returns - metadata about each. - - Current fields in metadata: - parent: ID of the parent image, or None if there is none - size: size in bytes as reported by docker - - :param tarfile_path: full path to a tarfile that is the product - of "docker save" - :type tarfile_path: basestring - - :return: A dictionary where keys are image IDs, and values are - dictionaries that contain the above-described metadata. - :rtype: dict - """ - metadata = {} - - with contextlib.closing(tarfile.open(tarfile_path)) as archive: - for member in archive.getmembers(): - # find the "json" files, which contain all image metadata - if os.path.basename(member.path) == 'json': - image_data = json.load(archive.extractfile(member)) - # At some point between docker 0.10 and 1.0, it changed behavior - # of whether these keys are capitalized or not. - image_id = image_data.get('id', image_data.get('Id')) - parent_id = image_data.get('parent', image_data.get('Parent')) - metadata[image_id] = { - 'parent': parent_id, - # image 511136ea does not have a Size attribute, which has - # caused problems during upload - 'size': image_data.get('Size'), - } - - return metadata - - -def get_tags(tarfile_path): - """ - returns a dictionary of docker tags, retrieved from the tarfile - - :param tarfile_path: full path to the tarfile - :type tarfile_path: basestring - """ - with contextlib.closing(tarfile.open(tarfile_path)) as archive: - repo_file = archive.extractfile('repositories') - repo_json = json.load(repo_file) - - if len(repo_json) != 1: - raise ValueError('pulp only supports one repo per tarfile') - - return repo_json.popitem()[1] - - -def get_ancestry(image_id, metadata): - """ - Given an image ID and metadata about each image, this calculates and returns - the ancestry list for that image. It walks the "parent" relationship in the - metadata to assemble the list, which is ordered with the child leaf at the - top. - - :param image_id: unique ID for a docker image - :type image_id: basestring - :param metadata: A dictionary where keys are image IDs, and values are - dictionaries that contain a key 'parent' with the ID - of a docker image - :type metadata dict - - :return: a tuple of image IDs where the first is the image_id passed in, - and each successive ID is the parent image of the ID that - proceeds it. - :rtype: tuple - """ - image_ids = [] - - while image_id: - image_ids.append(image_id) - image_id = metadata[image_id].get('parent') - - return tuple(image_ids) - - -def get_youngest_children(metadata): - """ - Given a full path to a tarfile, figure out which image IDs are leaf nodes, - aka the youngest children. - - :param metadata: a dictionary where keys are image IDs, and values are - dictionaries with keys "parent" and "size", containing - values for those two attributes as taken from the docker - image metadata. - :type metadata: dict - - :return: image IDs for the youngest docker images - :rtype: list - """ - image_ids = set(metadata.keys()) - for image_data in metadata.values(): - parent = image_data.get('parent') - if parent is not None: - try: - image_ids.remove(parent) - except KeyError: - # This can happen if an image is a parent of multiple child images, - # in which case this could be already removed from image_ids. - pass - - return list(image_ids) diff --git a/common/setup.py b/common/setup.py deleted file mode 100644 index e47a898a..00000000 --- a/common/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_docker_common', - version='0.1.0', - packages=find_packages(), - url='http://www.pulpproject.org', - license='GPLv2+', - author='Pulp Team', - author_email='pulp-list@redhat.com', - description='common code for pulp\'s docker image support', -) diff --git a/common/test/data/busyboxlight.tar b/common/test/data/busyboxlight.tar deleted file mode 100644 index f4c82e69..00000000 Binary files a/common/test/data/busyboxlight.tar and /dev/null differ diff --git a/common/test/unit/test_models.py b/common/test/unit/test_models.py deleted file mode 100644 index 75199010..00000000 --- a/common/test/unit/test_models.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from pulp_docker.common import models - - -class TestBasics(unittest.TestCase): - def test_init_info(self): - image = models.DockerImage('abc', 'xyz', 1024) - - self.assertEqual(image.image_id, 'abc') - self.assertEqual(image.parent_id, 'xyz') - self.assertEqual(image.size, 1024) - - def test_unit_key(self): - image = models.DockerImage('abc', 'xyz', 1024) - - self.assertEqual(image.unit_key, {'image_id': 'abc'}) - - def test_relative_path(self): - image = models.DockerImage('abc', 'xyz', 1024) - - self.assertEqual(image.relative_path, 'docker_image/abc') - - def test_metadata(self): - image = models.DockerImage('abc', 'xyz', 1024) - metadata = image.unit_metadata - - self.assertEqual(metadata.get('parent_id'), 'xyz') - self.assertEqual(metadata.get('size'), 1024) diff --git a/common/test/unit/test_tags.py b/common/test/unit/test_tags.py deleted file mode 100644 index 2338a270..00000000 --- a/common/test/unit/test_tags.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest - -from pulp_docker.common import constants, tags - - -class TestGenerateUpdatedTags(unittest.TestCase): - def test_generate_updated_tags(self): - scratchpad = {'tags': [{constants.IMAGE_TAG_KEY: 'tag1', - constants.IMAGE_ID_KEY: 'image1'}, - {constants.IMAGE_TAG_KEY: 'tag2', - constants.IMAGE_ID_KEY: 'image2'}, - {constants.IMAGE_TAG_KEY: 'tag-existing', - constants.IMAGE_ID_KEY: 'image-existing'}]} - new_tags = {'tag3': 'image3', 'tag-existing': 'image-new'} - update_tags = tags.generate_updated_tags(scratchpad, new_tags) - expected_update_tags = [{constants.IMAGE_TAG_KEY: 'tag1', - constants.IMAGE_ID_KEY: 'image1'}, - {constants.IMAGE_TAG_KEY: 'tag2', - constants.IMAGE_ID_KEY: 'image2'}, - {constants.IMAGE_TAG_KEY: 'tag-existing', - constants.IMAGE_ID_KEY: 'image-new'}, - {constants.IMAGE_TAG_KEY: 'tag3', - constants.IMAGE_ID_KEY: 'image3'}] - self.assertEqual(update_tags, expected_update_tags) - - def test_generate_updated_tags_empty_newtags(self): - scratchpad = {'tags': [{constants.IMAGE_TAG_KEY: 'tag1', - constants.IMAGE_ID_KEY: 'image1'}, - {constants.IMAGE_TAG_KEY: 'tag2', - constants.IMAGE_ID_KEY: 'image2'}, - {constants.IMAGE_TAG_KEY: 'tag-existing', - constants.IMAGE_ID_KEY: 'image-existing'}]} - new_tags = {} - update_tags = tags.generate_updated_tags(scratchpad, new_tags) - self.assertEqual(update_tags, scratchpad['tags']) diff --git a/common/test/unit/test_tarutils.py b/common/test/unit/test_tarutils.py deleted file mode 100644 index 61bd11e4..00000000 --- a/common/test/unit/test_tarutils.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -import unittest - -import mock - -from pulp_docker.common import tarutils - - -busybox_tar_path = os.path.join(os.path.dirname(__file__), '../data/busyboxlight.tar') - -# these are in correct ancestry order -busybox_ids = ( - '769b9341d937a3dba9e460f664b4f183a6cecdd62b337220a28b3deb50ee0a02', - '48e5f45168b97799ad0aafb7e2fef9fac57b5f16f6db7f67ba2000eb947637eb', - 'bf747efa0e2fa9f7c691588ce3938944c75607a7bb5e757f7369f86904d97c78', - '511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158', -) - -# This is a test metadata returned by tarutils.get_metadata when used on busybox.tar. -test_metadata_with_multiple_images_sharing_a_single_parent = { - u'150ddf0474655d12dcc79b0d5ee360dadcfba01e25d89dee71b4fed3d0c30fbe': { - 'parent': u'faab0acffc50714526b090fa60e0a55d79fc5b34fabbe6b964ca09cbb62f2026', - 'size': 0}, - u'4bf469521ee475733b739c3b15876b8d2b1102e3ce007f48a058e8830c9d2b47': { - 'parent': u'6c991eb934609424f761d3d0a7c79f4f72b76db286aa02e617659ac116aa7758', - 'size': 5454693}, - u'511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158': { - 'parent': None, - 'size': 0}, - u'6c991eb934609424f761d3d0a7c79f4f72b76db286aa02e617659ac116aa7758': { - 'parent': u'511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158', - 'size': 0}, - u'89aba41176b8f979bae09db1df5d6f3b58584318fce5d9e56b49c5a3e9700ab4': { - 'parent': u'4bf469521ee475733b739c3b15876b8d2b1102e3ce007f48a058e8830c9d2b47', - 'size': 0}, - u'8e36c99cfab52f0cf6f1aed7674cbdfe57e2ec8d29cdfdfac816b1d659d3ca9e': { - 'parent': u'900ce1b454ef7494e87709c727b8a456167eb7ea7bd202cb0d4b9911a6f05a5e', - 'size': 0}, - u'900ce1b454ef7494e87709c727b8a456167eb7ea7bd202cb0d4b9911a6f05a5e': { - 'parent': u'6c991eb934609424f761d3d0a7c79f4f72b76db286aa02e617659ac116aa7758', - 'size': 2433303}, - u'faab0acffc50714526b090fa60e0a55d79fc5b34fabbe6b964ca09cbb62f2026': { - 'parent': u'6c991eb934609424f761d3d0a7c79f4f72b76db286aa02e617659ac116aa7758', - 'size': 5609404} -} - - -class TestGetMetadata(unittest.TestCase): - def test_path_does_not_exist(self): - self.assertRaises(IOError, tarutils.get_metadata, '/a/b/c/d') - - def test_get_from_busybox(self): - metadata = tarutils.get_metadata(busybox_tar_path) - - self.assertEqual(set(metadata.keys()), set(busybox_ids)) - for i, image_id in enumerate(busybox_ids): - data = metadata[image_id] - if i == len(busybox_ids) - 1: - # make sure the base image has parent set to None - self.assertTrue(data['parent'] is None) - else: - # make sure all other layers have the correct parent - self.assertEqual(data['parent'], busybox_ids[i + 1]) - - # this image does not have a Size attribute in its json - if image_id.startswith('511136ea'): - self.assertTrue(data['size'] is None) - else: - self.assertTrue(isinstance(data['size'], int)) - - -class TestGetTags(unittest.TestCase): - def test_normal(self): - tags = tarutils.get_tags(busybox_tar_path) - - self.assertEqual(tags, {'latest': busybox_ids[0]}) - - @mock.patch('json.load', spec_set=True) - def test_no_repos(self, mock_load): - mock_load.return_value = {} - - self.assertRaises(ValueError, tarutils.get_tags, busybox_tar_path) - - -class TestGetAncestry(unittest.TestCase): - def test_from_busybox(self): - metadata = tarutils.get_metadata(busybox_tar_path) - ancestry = tarutils.get_ancestry(busybox_ids[0], metadata) - - self.assertEqual(ancestry, busybox_ids) - - -class TestGetYoungestChildren(unittest.TestCase): - def test_with_busybox_light(self): - metadata = tarutils.get_metadata(busybox_tar_path) - ret = tarutils.get_youngest_children(metadata) - - self.assertEqual(ret, [busybox_ids[0]]) - - def test_with_busybox(self): - ret = tarutils.get_youngest_children( - test_metadata_with_multiple_images_sharing_a_single_parent) - expected_youngest_children = [ - '150ddf0474655d12dcc79b0d5ee360dadcfba01e25d89dee71b4fed3d0c30fbe', - '89aba41176b8f979bae09db1df5d6f3b58584318fce5d9e56b49c5a3e9700ab4', - '8e36c99cfab52f0cf6f1aed7674cbdfe57e2ec8d29cdfdfac816b1d659d3ca9e'] - self.assertEqual(set(ret), set(expected_youngest_children)) diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 00000000..64a0c0d1 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,8 @@ +coverage +flake8 +flake8-docstrings +flake8-tuple +flake8-quotes +# pin pydocstyle until https://gitlab.com/pycqa/flake8-docstrings/issues/36 is resolved +pydocstyle<4 +requests diff --git a/doc_requirements.txt b/doc_requirements.txt new file mode 100644 index 00000000..edb3b88f --- /dev/null +++ b/doc_requirements.txt @@ -0,0 +1,2 @@ +sphinx<1.8.0 +towncrier diff --git a/docs/user-guide/Makefile b/docs/Makefile similarity index 86% rename from docs/user-guide/Makefile rename to docs/Makefile index b2bb2474..71be764a 100644 --- a/docs/user-guide/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -W -n SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build @@ -42,6 +42,7 @@ clean: -rm -rf $(BUILDDIR)/* html: + curl -o _static/api.json "http://localhost:24817/pulp/api/v3/docs/api.json?plugin=pulp_docker" $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." @@ -72,24 +73,6 @@ htmlhelp: @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PulpDockerTechnicalReference.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PulpDockerTechnicalReference.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PulpDockerTechnicalReference" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PulpDockerTechnicalReference" - @echo "# devhelp" - epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo diff --git a/docs/_scripts/base.sh b/docs/_scripts/base.sh new file mode 100755 index 00000000..b7a94369 --- /dev/null +++ b/docs/_scripts/base.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +echo "Setting environment variables for default hostname/port for the API and the Content app" +export BASE_ADDR=${BASE_ADDR:-http://localhost:24817} +export CONTENT_ADDR=${CONTENT_ADDR:-http://localhost:24816} + +# Necessary for `django-admin` +export DJANGO_SETTINGS_MODULE=pulpcore.app.settings + +# Poll a Pulp task until it is finished. +wait_until_task_finished() { + echo "Polling the task until it has reached a final state." + local task_url=$1 + while true + do + local response=$(http $task_url) + local state=$(jq -r .state <<< ${response}) + jq . <<< "${response}" + case ${state} in + failed|canceled) + echo "Task in final state: ${state}" + exit 1 + ;; + completed) + echo "$task_url complete." + break + ;; + *) + echo "Still waiting..." + sleep 2 + ;; + esac + done +} diff --git a/docs/_scripts/destination_repo.sh b/docs/_scripts/destination_repo.sh new file mode 100755 index 00000000..49f539d3 --- /dev/null +++ b/docs/_scripts/destination_repo.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +export DEST_REPO_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) + +echo "Create a second repository so we can add content to it." +export DEST_REPO_HREF=$(http POST $BASE_ADDR/pulp/api/v3/repositories/ name=$DEST_REPO_NAME \ + | jq -r '.pulp_href') + +echo "Inspect repository." +http $BASE_ADDR$DEST_REPO_HREF diff --git a/docs/_scripts/distribution.sh b/docs/_scripts/distribution.sh new file mode 100755 index 00000000..6d346259 --- /dev/null +++ b/docs/_scripts/distribution.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +export DIST_NAME='testing-hello' +export DIST_BASE_PATH='test' + +# Distributions are created asynchronously. +echo "Creating distribution \ + (name=$DIST_NAME, base_path=$DIST_BASE_PATH repository=$REPO_HREF)." +export TASK_HREF=$(http POST $BASE_ADDR/pulp/api/v3/distributions/docker/docker/ \ + name=$DIST_NAME \ + base_path=$DIST_BASE_PATH \ + repository=$REPO_HREF | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +echo "Setting DISTRIBUTION_HREF from the completed task." +# DISTRIBUTION_HREF is the pulp-api HREF, not the content app href +export DISTRIBUTION_HREF=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspecting Distribution." +http $BASE_ADDR$DISTRIBUTION_HREF diff --git a/docs/_scripts/docs_check.sh b/docs/_scripts/docs_check.sh new file mode 100755 index 00000000..a31bc34d --- /dev/null +++ b/docs/_scripts/docs_check.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# This script will execute the component scripts and ensure that the documented examples +# work as expected. + +# From the _scripts directory, run with `source docs_check_sync_publish.sh` (source to preserve the +# environment variables) +source base.sh + +# Check Sync +source repo.sh +source remote.sh +source sync.sh +source distribution.sh +source download_after_sync.sh + +# Check add/remove +source destination_repo.sh +source recursive_add_tag.sh +source recursive_remove_tag.sh + +# Check Copy +source destination_repo.sh +source manifest_copy.sh + +source destination_repo.sh +source tag_copy.sh + +# Check tag/untag +source image_tagging.sh +source download_after_tagging.sh +source image_untagging.sh diff --git a/docs/_scripts/download_after_sync.sh b/docs/_scripts/download_after_sync.sh new file mode 100755 index 00000000..ac45f8a0 --- /dev/null +++ b/docs/_scripts/download_after_sync.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +DOCKER_TAG='manifest_a' + +echo "Setting REGISTRY_PATH, which can be used directly with the Docker Client." +export REGISTRY_PATH=$(http $BASE_ADDR$DISTRIBUTION_HREF | jq -r '.registry_path') + +echo "Next we pull the image from pulp and run it." +echo "$REGISTRY_PATH:$DOCKER_TAG" +sudo docker run $REGISTRY_PATH:$DOCKER_TAG diff --git a/docs/_scripts/download_after_tagging.sh b/docs/_scripts/download_after_tagging.sh new file mode 100644 index 00000000..72cb6263 --- /dev/null +++ b/docs/_scripts/download_after_tagging.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +export TAG_NAME='custom_tag' + +export DIST_NAME='testing-tagging' +export DIST_BASE_PATH='tag' + +echo "Publishing the latest repository." +export TASK_URL=$(http POST $BASE_ADDR/pulp/api/v3/distributions/docker/docker/ \ + name=$DIST_NAME base_path=$DIST_BASE_PATH repository=$REPO_HREF \ + | jq -r '.task') + +wait_until_task_finished $BASE_ADDR$TASK_URL + +export DISTRIBUTION_HREF=$(http $BASE_ADDR$TASK_URL \ + | jq -r '.created_resources | first') +export REGISTRY_PATH=$(http $BASE_ADDR$DISTRIBUTION_HREF \ + | jq -r '.registry_path') + +echo "Pulling ${REGISTRY_PATH}:${TAG_NAME}." +docker run $REGISTRY_PATH:$TAG_NAME diff --git a/docs/_scripts/image_tagging.sh b/docs/_scripts/image_tagging.sh new file mode 100644 index 00000000..1f629978 --- /dev/null +++ b/docs/_scripts/image_tagging.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +export TAG_NAME='custom_tag' +export MANIFEST_DIGEST='sha256:21e3caae28758329318c8a868a80daa37ad8851705155fc28767852c73d36af5' + +echo "Tagging the manifest." +export TASK_URL=$(http POST $BASE_ADDR'/pulp/api/v3/docker/tag/' \ + repository=$REPO_HREF tag=$TAG_NAME digest=$MANIFEST_DIGEST \ + | jq -r '.task') + +wait_until_task_finished $BASE_ADDR$TASK_URL + +echo "Getting a reference to a newly created tag." +export CREATED_TAG=$(http $BASE_ADDR$TASK_URL \ + | jq -r '.created_resources | .[] | select(test("content"))') + +echo "Display properties of the created tag." +http $BASE_ADDR$CREATED_TAG diff --git a/docs/_scripts/image_untagging.sh b/docs/_scripts/image_untagging.sh new file mode 100644 index 00000000..2e0dd7c1 --- /dev/null +++ b/docs/_scripts/image_untagging.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +export TAG_NAME='custom_tag' + +echo "Untagging a manifest which is labeled with ${TAG_NAME}" +export TASK_URL=$(http POST $BASE_ADDR'/pulp/api/v3/docker/untag/' \ + repository=$REPO_HREF tag=$TAG_NAME \ + | jq -r '.task') + +wait_until_task_finished $BASE_ADDR$TASK_URL + +echo "Getting a reference to all removed tags." +export REPO_VERSION=$(http $BASE_ADDR$TASK_URL \ + | jq -r '.created_resources | first') +export REMOVED_TAGS=$(http $BASE_ADDR$REPO_VERSION \ + | jq -r '.content_summary | .removed | ."docker.tag" | .href') + +echo "List removed tags from the latest repository version." +http $BASE_ADDR$REMOVED_TAGS diff --git a/docs/_scripts/manifest_copy.sh b/docs/_scripts/manifest_copy.sh new file mode 100755 index 00000000..e73d076a --- /dev/null +++ b/docs/_scripts/manifest_copy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +echo "Create a task to copy all manifests from source to destination repo." +export TASK_HREF=$(http POST $BASE_ADDR'/pulp/api/v3/docker/manifests/copy/' \ + source_repository=$REPO_HREF \ + destination_repository=$DEST_REPO_HREF \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +export MANIFEST_COPY_VERSION=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$MANIFEST_COPY_VERSION diff --git a/docs/_scripts/recursive_add_tag.sh b/docs/_scripts/recursive_add_tag.sh new file mode 100755 index 00000000..54e187e5 --- /dev/null +++ b/docs/_scripts/recursive_add_tag.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +echo "Retrieve the href of Tag latest in the synced repository." +export TAG_HREF=$(http $BASE_ADDR'/pulp/api/v3/content/docker/tags/?repository_version='$REPOVERSION_HREF'&name=manifest_a' \ + | jq -r '.results | first | .pulp_href') + +echo "Create a task to recursively add a tag to the repo." +export TASK_HREF=$(http POST $BASE_ADDR'/pulp/api/v3/docker/recursive-add/' \ + repository=$DEST_REPO_HREF \ + content_units:="[\"$TAG_HREF\"]" \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +export ADDED_VERSION=$(http $BASE_ADDR$TASK_HREF| jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$ADDED_VERSION diff --git a/docs/_scripts/recursive_remove_tag.sh b/docs/_scripts/recursive_remove_tag.sh new file mode 100755 index 00000000..df80e46b --- /dev/null +++ b/docs/_scripts/recursive_remove_tag.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +echo "Create a task to recursively remove the same tag to the repo." +export TASK_HREF=$(http POST $BASE_ADDR'/pulp/api/v3/docker/recursive-remove/' \ + repository=$DEST_REPO_HREF \ + content_units:="[\"$TAG_HREF\"]" \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +export REMOVED_VERSION=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$REMOVED_VERSION diff --git a/docs/_scripts/remote.sh b/docs/_scripts/remote.sh new file mode 100755 index 00000000..74f504ae --- /dev/null +++ b/docs/_scripts/remote.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +echo "Creating a remote that points to an external source of container images." +http POST $BASE_ADDR/pulp/api/v3/remotes/docker/docker/ \ + name='my-hello-repo' \ + url='https://registry-1.docker.io' \ + upstream_name='pulp/test-fixture-1' + +echo "Export an environment variable for the new remote URI." +export REMOTE_HREF=$(http $BASE_ADDR/pulp/api/v3/remotes/docker/docker/ \ + | jq -r '.results[] | select(.name == "my-hello-repo") | .pulp_href') + +echo "Inspecting new Remote." +http $BASE_ADDR$REMOTE_HREF diff --git a/docs/_scripts/repo.sh b/docs/_scripts/repo.sh new file mode 100755 index 00000000..6b3d6456 --- /dev/null +++ b/docs/_scripts/repo.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +export REPO_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) + +echo "Creating a new repository named $REPO_NAME." +export REPO_HREF=$(http POST $BASE_ADDR/pulp/api/v3/repositories/ name=$REPO_NAME \ + | jq -r '.pulp_href') + +echo "Inspecting repository." +http $BASE_ADDR$REPO_HREF diff --git a/docs/_scripts/sync.sh b/docs/_scripts/sync.sh new file mode 100755 index 00000000..db764ca9 --- /dev/null +++ b/docs/_scripts/sync.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Create a task to sync the repository using the remote." +export TASK_HREF=$(http POST $BASE_ADDR$REMOTE_HREF'sync/' repository=$REPO_HREF mirror=False \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +echo "Set REPOVERSION_HREF from finished task." +export REPOVERSION_HREF=$(http $BASE_ADDR$TASK_HREF| jq -r '.created_resources | first') + +echo "Inspecting RepositoryVersion." +http $BASE_ADDR$REPOVERSION_HREF diff --git a/docs/_scripts/tag_copy.sh b/docs/_scripts/tag_copy.sh new file mode 100755 index 00000000..df3774c7 --- /dev/null +++ b/docs/_scripts/tag_copy.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +echo "Create a task to copy a tag to the repo." +export TASK_HREF=$(http POST $BASE_ADDR'/pulp/api/v3/docker/tags/copy/' \ + source_repository=$REPO_HREF \ + destination_repository=$DEST_REPO_HREF \ + names:="[\"manifest_a\"]" \ + | jq -r '.task') + +# Poll the task (here we use a function defined in docs/_scripts/base.sh) +wait_until_task_finished $BASE_ADDR$TASK_HREF + +# After the task is complete, it gives us a new repository version +export TAG_COPY_VERSION=$(http $BASE_ADDR$TASK_HREF | jq -r '.created_resources | first') + +echo "Inspect RepositoryVersion." +http $BASE_ADDR$TAG_COPY_VERSION diff --git a/docs/_static/api.json b/docs/_static/api.json new file mode 100644 index 00000000..0d5df3f9 --- /dev/null +++ b/docs/_static/api.json @@ -0,0 +1 @@ +{"swagger": "2.0", "info": {"title": "Pulp 3 API", "logo": {"url": "https://pulp.plan.io/attachments/download/517478/pulp_logo_word_rectangle.svg"}, "version": "v3"}, "host": "localhost:24817", "schemes": ["http"], "basePath": "/", "consumes": ["application/json"], "produces": ["application/json"], "securityDefinitions": {"Basic": {"type": "basic"}}, "security": [{"Basic": []}], "paths": {"/pulp/api/v3/content/docker/blobs/": {"get": {"operationId": "content_docker_blobs_list", "summary": "List blobs", "description": "ViewSet for Blobs.", "parameters": [{"name": "digest", "in": "query", "description": "Filter results where digest matches value", "required": false, "type": "string"}, {"name": "digest__in", "in": "query", "description": "Filter results where digest is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "repository_version", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_added", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_removed", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "media_type", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/Blob"}}}}}}, "tags": ["content: blobs"]}, "parameters": []}, "{blob_href}": {"get": {"operationId": "content_docker_blobs_read", "summary": "Inspect a blob", "description": "ViewSet for Blobs.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/Blob"}}}, "tags": ["content: blobs"]}, "parameters": [{"name": "blob_href", "in": "path", "description": "URI of Blob. e.g.: /pulp/api/v3/content/docker/blobs/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/content/docker/manifests/": {"get": {"operationId": "content_docker_manifests_list", "summary": "List manifests", "description": "ViewSet for Manifest.", "parameters": [{"name": "digest", "in": "query", "description": "Filter results where digest matches value", "required": false, "type": "string"}, {"name": "digest__in", "in": "query", "description": "Filter results where digest is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "repository_version", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_added", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_removed", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "media_type", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/Manifest"}}}}}}, "tags": ["content: manifests"]}, "parameters": []}, "{manifest_href}": {"get": {"operationId": "content_docker_manifests_read", "summary": "Inspect a manifest", "description": "ViewSet for Manifest.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/Manifest"}}}, "tags": ["content: manifests"]}, "parameters": [{"name": "manifest_href", "in": "path", "description": "URI of Manifest. e.g.: /pulp/api/v3/content/docker/manifests/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/content/docker/tags/": {"get": {"operationId": "content_docker_tags_list", "summary": "List tags", "description": "ViewSet for Tag.", "parameters": [{"name": "name", "in": "query", "description": "Filter results where name matches value", "required": false, "type": "string"}, {"name": "name__in", "in": "query", "description": "Filter results where name is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "repository_version", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_added", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_removed", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "media_type", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "digest", "in": "query", "description": "Multiple values may be separated by commas.", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/Tag"}}}}}}, "tags": ["content: tags"]}, "parameters": []}, "{tag_href}": {"get": {"operationId": "content_docker_tags_read", "summary": "Inspect a tag", "description": "ViewSet for Tag.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/Tag"}}}, "tags": ["content: tags"]}, "parameters": [{"name": "tag_href", "in": "path", "description": "URI of Tag. e.g.: /pulp/api/v3/content/docker/tags/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/distributions/docker/docker/": {"get": {"operationId": "distributions_docker_docker_list", "summary": "List docker distributions", "description": "The Docker Distribution will serve the latest version of a Repository if\n``repository`` is specified. The Docker Distribution will serve a specific\nrepository version if ``repository_version``. Note that **either**\n``repository`` or ``repository_version`` can be set on a Docker\nDistribution, but not both.", "parameters": [{"name": "name", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "name__in", "in": "query", "description": "Filter results where name is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "base_path", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "base_path__contains", "in": "query", "description": "Filter results where base_path contains value", "required": false, "type": "string"}, {"name": "base_path__icontains", "in": "query", "description": "Filter results where base_path contains value", "required": false, "type": "string"}, {"name": "base_path__in", "in": "query", "description": "Filter results where base_path is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/DockerDistribution"}}}}}}, "tags": ["distributions: docker"]}, "post": {"operationId": "distributions_docker_docker_create", "summary": "Create a docker distribution", "description": "Trigger an asynchronous create task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/DockerDistribution"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: docker"]}, "parameters": []}, "{docker_distribution_href}": {"get": {"operationId": "distributions_docker_docker_read", "summary": "Inspect a docker distribution", "description": "The Docker Distribution will serve the latest version of a Repository if\n``repository`` is specified. The Docker Distribution will serve a specific\nrepository version if ``repository_version``. Note that **either**\n``repository`` or ``repository_version`` can be set on a Docker\nDistribution, but not both.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/DockerDistribution"}}}, "tags": ["distributions: docker"]}, "put": {"operationId": "distributions_docker_docker_update", "summary": "Update a docker distribution", "description": "Trigger an asynchronous update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/DockerDistribution"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: docker"]}, "patch": {"operationId": "distributions_docker_docker_partial_update", "summary": "Partially update a docker distribution", "description": "Trigger an asynchronous partial update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/DockerDistribution"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: docker"]}, "delete": {"operationId": "distributions_docker_docker_delete", "summary": "Delete a docker distribution", "description": "Trigger an asynchronous delete task", "parameters": [], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: docker"]}, "parameters": [{"name": "docker_distribution_href", "in": "path", "description": "URI of Docker Distribution. e.g.: /pulp/api/v3/distributions/docker/docker/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/docker/manifests/copy/": {"post": {"operationId": "docker_manifests_copy_create", "description": "Trigger an asynchronous task to copy manifests", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/ManifestCopy"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["docker: copy"]}, "parameters": []}, "/pulp/api/v3/docker/recursive-add/": {"post": {"operationId": "docker_recursive-add_create", "description": "Trigger an asynchronous task to recursively add docker content.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RecursiveManage"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["docker: recursive-add"]}, "parameters": []}, "/pulp/api/v3/docker/recursive-remove/": {"post": {"operationId": "docker_recursive-remove_create", "description": "Trigger an asynchronous task to recursively remove docker content.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RecursiveManage"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["docker: recursive-remove"]}, "parameters": []}, "/pulp/api/v3/docker/tag/": {"post": {"operationId": "docker_tag_create", "description": "Trigger an asynchronous task to create a new repository", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/TagImage"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["docker: tag"]}, "parameters": []}, "/pulp/api/v3/docker/tags/copy/": {"post": {"operationId": "docker_tags_copy_create", "description": "Trigger an asynchronous task to copy tags", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/TagCopy"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["docker: copy"]}, "parameters": []}, "/pulp/api/v3/docker/untag/": {"post": {"operationId": "docker_untag_create", "description": "Trigger an asynchronous task to create a new repository", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/UnTagImage"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["docker: untag"]}, "parameters": []}, "/pulp/api/v3/remotes/docker/docker/": {"get": {"operationId": "remotes_docker_docker_list", "summary": "List docker remotes", "description": "Docker remotes represent an external repository that implements the Docker\nRegistry API. Docker remotes support deferred downloading by configuring\nthe ``policy`` field. ``on_demand`` and ``streamed`` policies can provide\nsignificant disk space savings.", "parameters": [{"name": "name", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "name__in", "in": "query", "description": "Filter results where name is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "_last_updated__lt", "in": "query", "description": "Filter results where _last_updated is less than value", "required": false, "type": "string"}, {"name": "_last_updated__lte", "in": "query", "description": "Filter results where _last_updated is less than or equal to value", "required": false, "type": "string"}, {"name": "_last_updated__gt", "in": "query", "description": "Filter results where _last_updated is greater than value", "required": false, "type": "string"}, {"name": "_last_updated__gte", "in": "query", "description": "Filter results where _last_updated is greater than or equal to value", "required": false, "type": "string"}, {"name": "_last_updated__range", "in": "query", "description": "Filter results where _last_updated is between two comma separated values", "required": false, "type": "string"}, {"name": "_last_updated", "in": "query", "description": "ISO 8601 formatted dates are supported", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/DockerRemote"}}}}}}, "tags": ["remotes: docker"]}, "post": {"operationId": "remotes_docker_docker_create", "summary": "Create a docker remote", "description": "Docker remotes represent an external repository that implements the Docker\nRegistry API. Docker remotes support deferred downloading by configuring\nthe ``policy`` field. ``on_demand`` and ``streamed`` policies can provide\nsignificant disk space savings.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/DockerRemote"}}], "responses": {"201": {"description": "", "schema": {"$ref": "#/definitions/DockerRemote"}}}, "tags": ["remotes: docker"]}, "parameters": []}, "{docker_remote_href}": {"get": {"operationId": "remotes_docker_docker_read", "summary": "Inspect a docker remote", "description": "Docker remotes represent an external repository that implements the Docker\nRegistry API. Docker remotes support deferred downloading by configuring\nthe ``policy`` field. ``on_demand`` and ``streamed`` policies can provide\nsignificant disk space savings.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/DockerRemote"}}}, "tags": ["remotes: docker"]}, "put": {"operationId": "remotes_docker_docker_update", "summary": "Update a docker remote", "description": "Trigger an asynchronous update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/DockerRemote"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: docker"]}, "patch": {"operationId": "remotes_docker_docker_partial_update", "summary": "Partially update a docker remote", "description": "Trigger an asynchronous partial update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/DockerRemote"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: docker"]}, "delete": {"operationId": "remotes_docker_docker_delete", "summary": "Delete a docker remote", "description": "Trigger an asynchronous delete task", "parameters": [], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: docker"]}, "parameters": [{"name": "docker_remote_href", "in": "path", "description": "URI of Docker Remote. e.g.: /pulp/api/v3/remotes/docker/docker/1/", "required": true, "type": "string"}]}, "{docker_remote_href}sync/": {"post": {"operationId": "remotes_docker_docker_sync", "description": "Trigger an asynchronous task to sync content.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RepositorySyncURL"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: docker"]}, "parameters": [{"name": "docker_remote_href", "in": "path", "description": "URI of Docker Remote. e.g.: /pulp/api/v3/remotes/docker/docker/1/", "required": true, "type": "string"}]}}, "definitions": {"Blob": {"required": ["artifact", "relative_path", "digest", "media_type"], "type": "object", "properties": {"_href": {"title": " href", "type": "string", "format": "uri", "readOnly": true}, "_created": {"title": " created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "_type": {"title": " type", "type": "string", "readOnly": true, "minLength": 1}, "artifact": {"title": "Artifact", "description": "Artifact file representing the physical content", "type": "string", "format": "uri"}, "relative_path": {"title": "Relative path", "description": "Path where the artifact is located relative to distributions base_path", "type": "string", "minLength": 1}, "digest": {"title": "Digest", "description": "sha256 of the Blob file", "type": "string", "minLength": 1}, "media_type": {"title": "Media type", "description": "Docker media type of the file", "type": "string", "minLength": 1}}}, "Manifest": {"required": ["artifact", "relative_path", "digest", "schema_version", "media_type", "listed_manifests", "config_blob", "blobs"], "type": "object", "properties": {"_href": {"title": " href", "type": "string", "format": "uri", "readOnly": true}, "_created": {"title": " created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "_type": {"title": " type", "type": "string", "readOnly": true, "minLength": 1}, "artifact": {"title": "Artifact", "description": "Artifact file representing the physical content", "type": "string", "format": "uri"}, "relative_path": {"title": "Relative path", "description": "Path where the artifact is located relative to distributions base_path", "type": "string", "minLength": 1}, "digest": {"title": "Digest", "description": "sha256 of the Manifest file", "type": "string", "minLength": 1}, "schema_version": {"title": "Schema version", "description": "Docker schema version", "type": "integer"}, "media_type": {"title": "Media type", "description": "Docker media type of the file", "type": "string", "minLength": 1}, "listed_manifests": {"description": "Manifests that are referenced by this Manifest List", "type": "array", "items": {"description": "Manifests that are referenced by this Manifest List", "type": "string", "format": "uri"}, "uniqueItems": true}, "config_blob": {"title": "Config blob", "description": "Blob that contains configuration for this Manifest", "type": "string", "format": "uri"}, "blobs": {"description": "Blobs that are referenced by this Manifest", "type": "array", "items": {"description": "Blobs that are referenced by this Manifest", "type": "string", "format": "uri"}, "uniqueItems": true}}}, "Tag": {"required": ["artifact", "relative_path", "name", "tagged_manifest"], "type": "object", "properties": {"_href": {"title": " href", "type": "string", "format": "uri", "readOnly": true}, "_created": {"title": " created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "_type": {"title": " type", "type": "string", "readOnly": true, "minLength": 1}, "artifact": {"title": "Artifact", "description": "Artifact file representing the physical content", "type": "string", "format": "uri"}, "relative_path": {"title": "Relative path", "description": "Path where the artifact is located relative to distributions base_path", "type": "string", "minLength": 1}, "name": {"title": "Name", "description": "Tag name", "type": "string", "minLength": 1}, "tagged_manifest": {"title": "Tagged manifest", "description": "Manifest that is tagged", "type": "string", "format": "uri"}}}, "DockerDistribution": {"required": ["base_path", "name"], "type": "object", "properties": {"content_guard": {"title": "Content guard", "description": "An optional content-guard.", "type": "string", "format": "uri", "x-nullable": true}, "repository_version": {"title": "Repository version", "description": "RepositoryVersion to be served", "type": "string", "format": "uri", "x-nullable": true}, "base_path": {"title": "Base path", "description": "The base (relative) path component of the published url. Avoid paths that overlap with other distribution base paths (e.g. \"foo\" and \"foo/bar\")", "type": "string", "maxLength": 255, "minLength": 1}, "_href": {"title": " href", "type": "string", "format": "uri", "readOnly": true}, "repository": {"title": "Repository", "description": "The latest RepositoryVersion for this Repository will be served.", "type": "string", "format": "uri", "x-nullable": true}, "name": {"title": "Name", "description": "A unique name. Ex, `rawhide` and `stable`.", "type": "string", "maxLength": 255, "minLength": 1}, "_created": {"title": " created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "registry_path": {"title": "Registry path", "description": "The Registry hostame:port/name/ to use with docker pull command defined by this distribution.", "type": "string", "readOnly": true, "minLength": 1}}}, "AsyncOperationResponse": {"required": ["task"], "type": "object", "properties": {"task": {"title": "Task", "description": "The href of the task.", "type": "string", "format": "uri"}}}, "ManifestCopy": {"required": ["destination_repository"], "type": "object", "properties": {"source_repository": {"title": "Repository", "description": "A URI of the repository to copy content from.", "type": "string", "format": "uri"}, "source_repository_version": {"title": "Source repository version", "description": "A URI of the repository version to copy content from.", "type": "string", "format": "uri"}, "destination_repository": {"title": "Repository", "description": "A URI of the repository to copy content to.", "type": "string", "format": "uri"}, "digests": {"description": "A list of manifest digests to copy.", "type": "array", "items": {"type": "string"}}, "media_types": {"description": "A list of media_types to copy.", "type": "array", "items": {"type": "string", "enum": ["application/vnd.docker.distribution.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json", "application/vnd.docker.distribution.manifest.list.v2+json"]}}}}, "RecursiveManage": {"required": ["repository"], "type": "object", "properties": {"repository": {"title": "Repository", "description": "A URI of the repository to add content.", "type": "string", "format": "uri"}, "content_units": {"description": "A list of content units to operate on.", "type": "array", "items": {"type": "string"}}}}, "TagImage": {"required": ["repository", "tag", "digest"], "type": "object", "properties": {"repository": {"title": "Repository", "description": "A URI of the repository.", "type": "string", "format": "uri"}, "tag": {"title": "Tag", "description": "A tag name", "type": "string", "minLength": 1}, "digest": {"title": "Digest", "description": "sha256 of the Manifest file", "type": "string", "minLength": 1}}}, "TagCopy": {"required": ["destination_repository"], "type": "object", "properties": {"source_repository": {"title": "Repository", "description": "A URI of the repository to copy content from.", "type": "string", "format": "uri"}, "source_repository_version": {"title": "Source repository version", "description": "A URI of the repository version to copy content from.", "type": "string", "format": "uri"}, "destination_repository": {"title": "Repository", "description": "A URI of the repository to copy content to.", "type": "string", "format": "uri"}, "names": {"description": "A list of tag names to copy.", "type": "array", "items": {"type": "string"}}}}, "UnTagImage": {"required": ["repository", "tag"], "type": "object", "properties": {"repository": {"title": "Repository", "description": "A URI of the repository.", "type": "string", "format": "uri"}, "tag": {"title": "Tag", "description": "A tag name", "type": "string", "minLength": 1}}}, "DockerRemote": {"required": ["name", "url", "upstream_name"], "type": "object", "properties": {"_href": {"title": " href", "type": "string", "format": "uri", "readOnly": true}, "_created": {"title": " created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "_type": {"title": " type", "type": "string", "readOnly": true, "minLength": 1}, "name": {"title": "Name", "description": "A unique name for this remote.", "type": "string", "minLength": 1}, "url": {"title": "Url", "description": "The URL of an external content source.", "type": "string", "minLength": 1}, "ssl_ca_certificate": {"title": "Ssl ca certificate", "description": "A string containing the PEM encoded CA certificate used to validate the server certificate presented by the remote server. All new line characters must be escaped. Returns SHA256 sum on GET.", "type": "string", "minLength": 1, "x-nullable": true}, "ssl_client_certificate": {"title": "Ssl client certificate", "description": "A string containing the PEM encoded client certificate used for authentication. All new line characters must be escaped. Returns SHA256 sum on GET.", "type": "string", "minLength": 1, "x-nullable": true}, "ssl_client_key": {"title": "Ssl client key", "description": "A PEM encoded private key used for authentication. Returns SHA256 sum on GET.", "type": "string", "minLength": 1, "x-nullable": true}, "ssl_validation": {"title": "Ssl validation", "description": "If True, SSL peer validation must be performed.", "type": "boolean"}, "proxy_url": {"title": "Proxy url", "description": "The proxy URL. Format: scheme://user:password@host:port", "type": "string", "minLength": 1, "x-nullable": true}, "username": {"title": "Username", "description": "The username to be used for authentication when syncing.", "type": "string", "minLength": 1, "x-nullable": true}, "password": {"title": "Password", "description": "The password to be used for authentication when syncing.", "type": "string", "minLength": 1, "x-nullable": true}, "_last_updated": {"title": " last updated", "description": "Timestamp of the most recent update of the remote.", "type": "string", "format": "date-time", "readOnly": true}, "download_concurrency": {"title": "Download concurrency", "description": "Total number of simultaneous connections.", "type": "integer", "minimum": 1}, "policy": {"title": "Policy", "description": "\n immediate - All manifests and blobs are downloaded and saved during a sync.\n on_demand - Only tags and manifests are downloaded. Blobs are not\n downloaded until they are requested for the first time by a client.\n streamed - Blobs are streamed to the client with every request and never saved.\n ", "type": "string", "enum": ["immediate", "on_demand", "streamed"], "default": "immediate"}, "upstream_name": {"title": "Upstream name", "description": "Name of the upstream repository", "type": "string", "minLength": 1}, "whitelist_tags": {"title": "Whitelist tags", "description": "A comma separated string of tags to sync.\n Example:\n\n latest,1.27.0\n ", "type": "string", "minLength": 1, "x-nullable": true}}}, "RepositorySyncURL": {"required": ["repository"], "type": "object", "properties": {"repository": {"title": "Repository", "description": "A URI of the repository to be synchronized.", "type": "string", "format": "uri"}, "mirror": {"title": "Mirror", "description": "If ``True``, synchronization will remove all content that is not present in the remote repository. If ``False``, sync will be additive only.", "type": "boolean", "default": false}}}}, "tags": [{"name": "content: blobs", "x-displayName": "Content: Blobs"}, {"name": "content: manifests", "x-displayName": "Content: Manifests"}, {"name": "content: tags", "x-displayName": "Content: Tags"}, {"name": "distributions: docker", "x-displayName": "Distributions: Docker"}, {"name": "docker: copy", "x-displayName": "Docker: Copy"}, {"name": "docker: recursive-add", "x-displayName": "Docker: Recursive-Add"}, {"name": "docker: recursive-remove", "x-displayName": "Docker: Recursive-Remove"}, {"name": "docker: tag", "x-displayName": "Docker: Tag"}, {"name": "docker: untag", "x-displayName": "Docker: Untag"}, {"name": "remotes: docker", "x-displayName": "Remotes: Docker"}]} \ No newline at end of file diff --git a/docs/_templates/restapi.html b/docs/_templates/restapi.html new file mode 100644 index 00000000..0d666805 --- /dev/null +++ b/docs/_templates/restapi.html @@ -0,0 +1,24 @@ + + + + REST API for Pulp 3 Docker Plugin + + + + + + + + + + + + + diff --git a/docs/changes.rst b/docs/changes.rst new file mode 100644 index 00000000..d391fe90 --- /dev/null +++ b/docs/changes.rst @@ -0,0 +1,5 @@ +.. _pulp-docker-changes: + +.. include:: ../CHANGES.rst + +.. include:: ../HISTORY.rst diff --git a/docs/user-guide/conf.py b/docs/conf.py similarity index 83% rename from docs/user-guide/conf.py rename to docs/conf.py index 224ca9fe..823a4cf2 100644 --- a/docs/user-guide/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Pulp Docker documentation build configuration file, created by -# sphinx-quickstart on Wed May 21 09:44:51 2014. +# Pulp Docker Support documentation build configuration file, created by modifying a config +# built by sphinx-quickstart on Thu Nov 20 17:53:15 2014. # # This file is execfile()d with the current directory set to its containing dir. # @@ -11,7 +11,16 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os +import sys +sys.path.insert(0, os.path.abspath('..')) # noqa + +import pulp_docker + +try: + import sphinx_rtd_theme +except ImportError: + sphinx_rtd_theme = False # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -25,7 +34,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] +extensions = ['sphinx.ext.extlinks'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -40,17 +49,17 @@ master_doc = 'index' # General information about the project. -project = u'Pulp Docker' -copyright = u'2014, Pulp Team' +project = u'Pulp Docker Support' +copyright = u'2012-2019, Pulp Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.1' +version = pulp_docker.__version__ # The full version, including alpha/beta/rc tags. -release = '0.1' +release = pulp_docker.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -91,7 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'sphinx_rtd_theme' if sphinx_rtd_theme else 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -120,7 +129,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] if sphinx_rtd_theme else [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -136,6 +145,9 @@ # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} +html_additional_pages = {'restapi': 'restapi.html'} + +html_static_path = ['_static'] # If false, no module index is generated. #html_domain_indices = True @@ -163,10 +175,6 @@ # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None -# Output file base name for HTML help builder. -htmlhelp_basename = 'PulpDockerdoc' - - # -- Options for LaTeX output -------------------------------------------------- latex_elements = { @@ -180,13 +188,6 @@ #'preamble': '', } -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PulpDocker.tex', u'Pulp Docker Documentation', - u'Pulp Team', 'manual'), -] - # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None @@ -208,30 +209,9 @@ #latex_domain_indices = True -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pulpdocker', u'Pulp Docker Documentation', - [u'Pulp Team'], 1) -] - # If true, show URL addresses after external links. #man_show_urls = False - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'PulpDocker', u'Pulp Docker Documentation', - u'Pulp Team', 'PulpDocker', 'One line description of project.', - 'Miscellaneous'), -] - # Documents to append as an appendix to all manuals. #texinfo_appendices = [] diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 00000000..e582053e --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/ddoc.sh b/docs/ddoc.sh new file mode 100755 index 00000000..66983e51 --- /dev/null +++ b/docs/ddoc.sh @@ -0,0 +1,8 @@ +source ~/.bashrc +prestart +cd ~/devel/pulp_docker/docs/ +sleep 2 +make clean +make html +cd ~/devel/pulp_docker/docs/_build/html +python2 -m SimpleHTTPServer 1234 diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..0762fa94 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,56 @@ +Pulp Docker Plugin +================== + +The ``pulp_docker`` plugin extends `pulpcore `__ to support +hosting containers and container metadata, supporting ``docker pull`` and ``podman pull``. + +If you are just getting started, we recommend getting to know the :doc:`basic +workflows`. + +Features +-------- + +* :ref:`Synchronize ` from a Docker registry that has basic or token auth +* :ref:`Create Versioned Repositories ` so every operation is a restorable snapshot +* :ref:`Download content on-demand ` when requested by clients to reduce disk space +* :ref:`Perform docker/podman pull ` from a docker distribution served by Pulp +* De-duplication of all saved content +* Host content either `locally or on S3 `_ + + +How to use these docs +--------------------- + +The documentation here should be considered **the primary documentation for managing container +related content**. All relevent workflows are covered here, with references to some pulpcore +supplemental docs. Users may also find `pulpcore's conceptual docs +`_ useful. + +This documentation falls into two main categories: + + 1. :ref:`workflows-index` show the **major features** of the docker plugin, with links to + reference docs. + 2. `REST API Docs `_ are automatically generated and provide more detailed + information for each **minor feature**, including all fields and options. + +Container Workflows +------------------- + +.. toctree:: + :maxdepth: 1 + + installation + workflows/index + restapi/index + changes + contributing + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 00000000..3738f17a --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,67 @@ + +User Setup +========== + +Ansible Installer (Recommended) +------------------------------- + +We recommend that you install `pulpcore` and `pulp-docker` together using the `Ansible installer +`_. If you install this way, pulpcore +installation and all the following steps will be done for you. + +Install ``pulpcore`` +-------------------- + +Follow the `installation +instructions `__ +provided with pulpcore. + +Install plugin +-------------- + +This document assumes that you have +`installed pulpcore `_ +into a the virtual environment ``pulpvenv``. + +Users should install from **either** PyPI or source. + +From PyPI +********* + +.. code-block:: bash + + sudo -u pulp -i + source ~/pulpvenv/bin/activate + pip install pulp-docker + + +Install ``pulp_docker`` from source +*********************************** + +.. code-block:: bash + + sudo -u pulp -i + source ~/pulpvenv/bin/activate + cd pulp_docker + pip install -e . + django-admin runserver 24817 + +Make and Run Migrations +----------------------- + +.. code-block:: bash + + export DJANGO_SETTINGS_MODULE=pulpcore.app.settings + django-admin makemigrations docker + django-admin migrate docker + +Run Services +------------ + +.. code-block:: bash + + pulp-manager runserver + gunicorn pulpcore.content:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 + sudo systemctl restart pulpcore-resource-manager + sudo systemctl restart pulpcore-worker@1 + sudo systemctl restart pulpcore-worker@2 diff --git a/docs/restapi/index.rst b/docs/restapi/index.rst new file mode 100644 index 00000000..48279f52 --- /dev/null +++ b/docs/restapi/index.rst @@ -0,0 +1,9 @@ +REST API +======== + +Pulpcore Reference: `pulpcore REST documentation `_. + +Pulp Docker Endpoints +--------------------- + +REST API Reference `docker REST documentation <../restapi.html>`_ diff --git a/docs/tech-reference/Makefile b/docs/tech-reference/Makefile deleted file mode 100644 index b2bb2474..00000000 --- a/docs/tech-reference/Makefile +++ /dev/null @@ -1,153 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PulpDockerTechnicalReference.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PulpDockerTechnicalReference.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PulpDockerTechnicalReference" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PulpDockerTechnicalReference" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/tech-reference/conf.py b/docs/tech-reference/conf.py deleted file mode 100644 index 4079e273..00000000 --- a/docs/tech-reference/conf.py +++ /dev/null @@ -1,240 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Pulp Docker Technical Reference documentation build configuration file, created by -# sphinx-quickstart on Thu Apr 3 16:59:06 2014. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Pulp Docker Technical Reference' -copyright = u'2014, Pulp Team' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.1' -# The full version, including alpha/beta/rc tags. -release = '0.1.0' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'PulpDockerTechnicalReferencedoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - #'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PulpDockerTechnicalReference.tex', u'Pulp Docker Technical Reference Documentation', - u'Pulp Team', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pulpdockertechnicalreference', u'Pulp Docker Technical Reference Documentation', - [u'Pulp Team'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'PulpDockerTechnicalReference', u'Pulp Docker Technical Reference Documentation', - u'Pulp Team', 'PulpDockerTechnicalReference', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' diff --git a/docs/tech-reference/distributor.rst b/docs/tech-reference/distributor.rst deleted file mode 100644 index 45505ac3..00000000 --- a/docs/tech-reference/distributor.rst +++ /dev/null @@ -1,125 +0,0 @@ -Distributor Configuration -========================= - - -Web Distributor ---------------- - -Type ID: ``docker_distributor_web`` - -The Web distributor is used to publish a docker repository in a way that can be consumed -and served by Crane directly. By default the :ref:`redirect file ` is stored as -``/var/lib/pulp/published/docker/app/.json``, and the repo data itself is stored in -``/var/lib/pulp/published/docker/web//``. - -The global configuration file for the docker_web_distributor plugin -can be found in ``/etc/pulp/server/plugin.conf.d/docker_distributor.json``. - -All values from the global configuration can be overridden on the local config. - -Supported keys -^^^^^^^^^^^^^^ - -``docker_publish_directory`` - The publish directory used for this distributor. The web server should be configured to serve - /web. The default value is ``/var/lib/pulp/published/docker``. - -``protected`` - if "true" requests for this repo will be checked for an entitlement certificate authorizing - the server url for this repository; if "false" no authorization checking will be done. - This defaults to true. - -``redirect-url`` - The server URL that will be used when generating the redirect map for connecting the docker - API to the location the content is stored. The value defaults to - ``https:///pulp/docker/``. - -``repo-registry-id`` - The name that should be used for the repository when it is served by Crane. If specified - it will be used for the ``repository`` field in the :ref:`redirect file `. - If a value is not specified, then repository id is used. - - -Export Distributor ------------------- - -Type ID: ``docker_distributor_export`` - -The export distributor is used to save the contents of a publish into a tar file that can be -moved easily for instances where Crane is running on a different server than your pulp instance. -By default the :ref:`redirect file ` is stored in the root of the tar file as -``.json``, and the repo data itself is stored in the ``//`` sub directory of -the tar file. - -The global configuration file for the docker_export_distributor plugin -can be found in ``/etc/pulp/server/plugin.conf.d/docker_distributor_export.json``. - -All values from the global configuration can be overridden on the local config. - -Supported keys -^^^^^^^^^^^^^^ - -``docker_publish_directory`` - The publish directory used for this distributor. The web server should be configured to serve - /export. The default value is ``/var/lib/pulp/published/docker``. - -``export_file`` - The fully qualified path and name of the tar file that will be created by the export. - This defaults to ``/export/repo/.tar`` - -``protected`` - if "true" requests for this repo will be checked for an entitlement certificate authorizing - the server url for this repository; if "false" no authorization checking will be done. - -``redirect-url`` - The URL where image files for this repository are served. Crane will join this URL with - ``/`` - -``repo-registry-id`` - The name that should be used for the repository when it is served by Crane. If specified - it will be used for the ``repository`` field in the :ref:`redirect file `. - If a value is not specified, then repository id is used. - - -.. _redirect_file: - -Redirect File -------------- - -The distributors generate a json file with the details of the repository contents. - -The file is JSON formatted with the following keys - -* **type** *(string)* - the type of the file. This will always be "pulp-docker-redirect" -* **version** *(int)* - version of the format for the file. Currently version 1 -* **repository** *(string)* - the name of the repository this file is describing -* **repo-registry-id** *(string)* - the name that will be used for this repository in the Docker - registry -* **url** *(string)* - the url for access to the repositories content -* **protected** *(bool)* - whether or not the repository should be protected by an entitlement - certificate. -* **images** *(array)* - an array of objects describing each image/layer in the repository - - * **id** *(str)* - the image id for the image - -* **tags** *(obj)* - an object containing key, value paris of "tag-name":"image-id" - -Example Redirect File Contents:: - - { - "type":"pulp-docker-redirect", - "version":1, - "repository":"docker", - "repo-registry-id":"redhat/docker", - "url":"http://www.foo.com/docker", - "protected": true, - "images":[ - {"id":"48e5f45168b97799ad0aafb7e2fef9fac57b5f16f6db7f67ba2000eb947637eb"}, - {"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}, - {"id":"769b9341d937a3dba9e460f664b4f183a6cecdd62b337220a28b3deb50ee0a02"}, - {"id":"bf747efa0e2fa9f7c691588ce3938944c75607a7bb5e757f7369f86904d97c78"} - ], - "tags": {"latest": "769b9341d937a3dba9e460f664b4f183a6cecdd62b337220a28b3deb50ee0a02"} - } - - diff --git a/docs/tech-reference/importer.rst b/docs/tech-reference/importer.rst deleted file mode 100644 index ce82ad46..00000000 --- a/docs/tech-reference/importer.rst +++ /dev/null @@ -1,21 +0,0 @@ -Importer -======== - -ID: ``docker_importer`` - -Configuration -------------- - -The following options are available to the docker importer configuration. - -``mask_id`` - Supported only as an override config option to a repository upload command, when - this option is used, the upload command will skip adding given image and - any ancestors of that image to the repository. - -``feed`` - The URL for the docker repository to import images from - -``upstream_name`` - The name of the repository to import from the upstream repository - diff --git a/docs/tech-reference/index.rst b/docs/tech-reference/index.rst deleted file mode 100644 index 4523ef9e..00000000 --- a/docs/tech-reference/index.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. Pulp Docker Technical Reference documentation master file, created by - sphinx-quickstart on Thu Apr 3 16:59:06 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Pulp Docker Technical Reference's documentation! -=========================================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - importer - distributor - tags - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/tech-reference/tags.rst b/docs/tech-reference/tags.rst deleted file mode 100644 index 5fed0b99..00000000 --- a/docs/tech-reference/tags.rst +++ /dev/null @@ -1,20 +0,0 @@ -Tags -==== - -Tags on images are managed via the repository object. In the ``tags`` sub object of the -``scratchpad`` object, a list of key value pairs for each tag & Image ID are stored as -shown below. - -Example Repository Object:: - - { - ... - "scratchpad": { - ... - "tags": [ - { "tag": "latest", - "image_id": "48e5f45168b97799ad0aafb7e2fef9fac57b5f16f6db7f67ba2000eb947637eb"} - ] - } - - diff --git a/docs/user-guide/concepts.rst b/docs/user-guide/concepts.rst deleted file mode 100644 index d14c9e8c..00000000 --- a/docs/user-guide/concepts.rst +++ /dev/null @@ -1,19 +0,0 @@ -Concepts -======== - -Repository and Tags -------------------- - -A docker repository is a collection of images that can have tags. A pulp -repository likewise is a collection of docker images. Tags are a property of the -repository and can be modified with the command ``pulp-admin docker repo update`` -and its ``--tag`` option. - -Upload ------- - -An upload operation potentially includes multiple layers. When doing a -``docker save``, a tarball is created with the requested repository and all of -its ancestor layers. When uploading that tarball to pulp, each layer will be -added to the repository as a unit. The tags will also be added to the -repository, overwriting any previous tags of the same name. diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst deleted file mode 100644 index e2239ab2..00000000 --- a/docs/user-guide/index.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Pulp Docker documentation master file, created by - sphinx-quickstart on Wed May 21 09:44:51 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Pulp Docker Support -=================== - -This project adds support to Pulp for managing Docker images. - -Contents: - -.. toctree:: - :maxdepth: 2 - - installation - concepts - recipes - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/user-guide/installation.rst b/docs/user-guide/installation.rst deleted file mode 100644 index 1976e542..00000000 --- a/docs/user-guide/installation.rst +++ /dev/null @@ -1,26 +0,0 @@ -Installation -============ - -.. _Pulp User Guide: http://pulp-user-guide.readthedocs.org - -Prerequisites -------------- - -The only requirement is to meet the prerequisites of the Pulp Platform. Please -see the `Pulp User Guide`_ for prerequisites including repository setup. - -Development ------------ - -The only way to install docker support currently is to setup a development -environment. Installation through RPMs will come at a later time. - -:: - - git clone https://github.com/pulp/pulp_docker.git - cd pulp_docker - sudo ./manage_setup_pys.sh develop - sudo ./pulp-dev.py -I - sudo -u apache pulp-manage-db - -Then restart each pulp component, as documented in the `Pulp User Guide`_. diff --git a/docs/user-guide/recipes.rst b/docs/user-guide/recipes.rst deleted file mode 100644 index 57d650dc..00000000 --- a/docs/user-guide/recipes.rst +++ /dev/null @@ -1,269 +0,0 @@ -Recipes -======= - -.. _Crane: https://github.com/pulp/crane - -Configuring `Crane`_ with pulp_docker -------------------------------------- -The Crane project can be used to make docker repositories hosted by Pulp available -to the docker client. This allows a ``docker pull`` to be performed against data -that is published by the Pulp server. - -If `Crane`_ is being run on the same server that is running Pulp, there is one setting that -must be configured in Crane in order for it to find the information that is published by Pulp. -In the /etc/crane.conf the ``data_dir`` parameter must be set to the location that the pulp publish -is placing metadata files. By default this is the ``/var/lib/pulp/published/docker/app/`` -directory. Crane will check the ``data_dir`` for updates periodically. -Full documentation for /etc/crane.conf can be found in the `Crane`_ readme. - - -Upload To Pulp --------------- - -To upload a docker image to pulp, first you must save its repository with docker. -Note that the below command saves all of the images and tags in the ``busybox`` -repository to a tarball:: - - $ sudo docker pull busybox - $ sudo docker save busybox > busybox.tar - -Then create a pulp repository and run an upload command with ``pulp-admin``:: - - $ pulp-admin docker repo create --repo-id=busybox - Repository [busybox] successfully created - - $ pulp-admin docker repo uploads upload --repo-id=busybox -f busybox.tar - +----------------------------------------------------------------------+ - Unit Upload - +----------------------------------------------------------------------+ - - Extracting necessary metadata for each request... - [==================================================] 100% - Analyzing: busybox.tar - ... completed - - Creating upload requests on the server... - [==================================================] 100% - Initializing: busybox.tar - ... completed - - Starting upload of selected units. If this process is stopped through ctrl+c, - the uploads will be paused and may be resumed later using the resume command or - cancelled entirely using the cancel command. - - Uploading: busybox.tar - [==================================================] 100% - 2825216/2825216 bytes - ... completed - - Importing into the repository... - This command may be exited via ctrl+c without affecting the request. - - - [\] - Running... - - Task Succeeded - - - Deleting the upload request... - ... completed - - -There are now four new images in the pulp repository:: - - $ pulp-admin docker repo list - +----------------------------------------------------------------------+ - Docker Repositories - +----------------------------------------------------------------------+ - - Id: busybox - Display Name: busybox - Description: None - Content Unit Counts: - Docker Image: 4 - - -During an image upload, you can specify the id of an ancestor image -that should not be uploaded to the repository. In this case, the masked ancestor -and any ancestors of that image will not be imported:: - - $ pulp-admin docker repo create --repo-id tutorial - Repository [tutorial] successfully created - - $ pulp-admin docker repo uploads upload --repo-id tutorial - -f /home/skarmark/git/pulp1/pulp/tutorial.tar - --mask-ancestor-id 'f38e479062c4953de709cc7f08fa8f85bec6bc5d01f03e340f7caf2990e8efd1' - +----------------------------------------------------------------------+ - Unit Upload - +----------------------------------------------------------------------+ - - Extracting necessary metadata for each request... - [==================================================] 100% - Analyzing: tutorial.tar - ... completed - - Creating upload requests on the server... - [==================================================] 100% - Initializing: tutorial.tar - ... completed - - Starting upload of selected units. If this process is stopped through ctrl+c, - the uploads will be paused and may be resumed later using the resume command or - cancelled entirely using the cancel command. - - Uploading: tutorial.tar - [==================================================] 100% - 353358336/353358336 bytes - ... completed - - Importing into the repository... - This command may be exited via ctrl+c without affecting the request. - - - [\] - Running... - - Task Succeeded - - - Deleting the upload request... - ... completed - -There are now only two images imported into the pulp repository, instead of five total images -in the tar file:: - - $ pulp-admin docker repo list - +----------------------------------------------------------------------+ - Docker Repositories - +----------------------------------------------------------------------+ - - Id: tutorial - Display Name: tutorial - Description: None - Content Unit Counts: - Docker Image: 2 - - -Publish -------- - -The ``busybox`` repository uploaded above can be published for use with `Crane`_. - -First the docker repository name must be specified, which can -be different than the ``repo_id``. The repository name should usually have a -namespace, a ``/``, and then a name. The command below sets the repository name -to ``pulpdemo/busybox``:: - - $ pulp-admin docker repo update --repo-id=busybox --repo-registry-id=pulpdemo/busybox - This command may be exited via ctrl+c without affecting the request. - - - [\] - Running... - Updating distributor: docker_web_distributor_name_cli - - Task Succeeded - - - - [\] - Running... - Updating distributor: docker_export_distributor_name_cli - - Task Succeeded - -Then a publish operation can be executed:: - - $ pulp-admin docker repo publish run --repo-id=busybox - +----------------------------------------------------------------------+ - Publishing Repository [busybox] - +----------------------------------------------------------------------+ - - This command may be exited via ctrl+c without affecting the request. - - - Publishing Image Files. - [==================================================] 100% - 4 of 4 items - ... completed - - Making files available via web. - [-] - ... completed - - - Task Succeeded - - -`Crane`_ can now be run on the same machine serving the docker repository through -its docker-registry-like read-only API. - -Export ------- - -The ``busybox`` repository can also be exported for a case where `Crane`_ will -be run on a different machine, or the image files will be hosted by another -service:: - - $ pulp-admin docker repo export run --repo-id=busybox - +----------------------------------------------------------------------+ - Publishing Repository [busybox] - +----------------------------------------------------------------------+ - - This command may be exited via ctrl+c without affecting the request. - - - Publishing Image Files. - [==================================================] 100% - 4 of 4 items - ... completed - - Saving tar file. - [-] - ... completed - - - Task Succeeded - -This produces a tarball at ``/var/lib/pulp/published/docker/export/repo/busybox.tar`` -which contains both a JSON file for use with crane, and the static image files -to which crane will redirect requests. See the `Crane`_ documentation for how -to use that tarball. - -Sync ------- - -The pulp-docker plugin supports syncing from upstream repositories as of version 0.2.1. For example:: - - $ pulp-admin docker repo create synctest --feed=https://index.docker.io --upstream-name=busybox - Repository [synctest] successfully created - - $ pulp-admin docker repo sync run --repo-id synctest - +----------------------------------------------------------------------+ - Synchronizing Repository [synctest] - +----------------------------------------------------------------------+ - - This command may be exited via ctrl+c without affecting the request. - - - Retrieving metadata - [\] - ... completed - - Copying units already in pulp - [-] - ... completed - - Downloading remote files - [-] - ... completed - - Saving images and tags - [-] - ... completed - - - Task Succeeded - -Once this is complete, the data in the remote repository is now in your local Pulp instance. diff --git a/docs/workflows/authentication.rst b/docs/workflows/authentication.rst new file mode 100644 index 00000000..2e04f465 --- /dev/null +++ b/docs/workflows/authentication.rst @@ -0,0 +1,97 @@ +.. _authentication: + +Registry Token Authentication +============================= + +Pulp registry supports the `token authentication `_. +This enables users to pull content with an authorized access. A token server grants access based on the +user's privileges and current scope. + +The feature is enabled by default. However, it is required to define the following settings first: + + - **A fully qualified domain name of a token server**. The token server is responsible for generating + Bearer tokens. Append the constant ``TOKEN_SERVER`` to the settings file ``pulp_docker/app/settings.py``. + - **A token signature algorithm**. A particular signature algorithm can be chosen only from the list of + `supported algorithms `_. + Pulp uses exclusively asymmetric cryptography to sign and validate tokens. Therefore, it is possible + only to choose from the algorithms, such as ES256, RS256, or PS256. Append the the constant + ``TOKEN_SIGNATURE_ALGORITHM`` with a selected algorithm to the settings file. + - **Paths to secure keys**. These keys are going to be used for a signing and validation of tokens. + Remember that the keys have to be specified in the **PEM format**. To generate keys, one could use + the openssl utility. In the following example, the utility is used to generate keys with the algorithm + ES256. + + 1. Generate a private key:: + + $ openssl ecparam -genkey -name prime256v1 -noout -out /tmp/private_key.pem + + 2. Generate a public key out of the private key:: + + $ openssl ec -in /tmp/private_key.pem -pubout -out /tmp/public_key.pem + +Below is provided and example of the settings file: + +.. code-block:: python + + TOKEN_SERVER = "localhost:24816/token" + TOKEN_SIGNATURE_ALGORITHM = 'ES256' + PUBLIC_KEY_PATH = '/tmp/public_key.pem' + PRIVATE_KEY_PATH = '/tmp/private_key.pem' + +To learn more about Pulp settings, take a look at `Configuration +`_. + +Restart Pulp services in order to reload the updated settings. Pulp will fetch a domain for the token +server and will initialize all handlers according to that. Check if the token authentication was +successfully configured by initiating the following set of commands in your environment:: + + $ http 'http://localhost:24816/v2/' + + HTTP/1.1 401 Access to the requested resource is not authorized. A provided Bearer token is invalid. + Content-Length: 92 + Content-Type: text/plain; charset=utf-8 + Date: Mon, 14 Oct 2019 16:46:48 GMT + Docker-Distribution-API-Version: registry/2.0 + Server: Python/3.7 aiohttp/3.6.1 + Www-Authenticate: Bearer realm="http://localhost:24816/token",service="localhost:24816" + + 401: Access to the requested resource is not authorized. A provided Bearer token is invalid. + +Send a request to a specified realm:: + + $ http 'http://localhost:24816/token?service=localhost:24816' + + HTTP/1.1 200 OK + Content-Length: 566 + Content-Type: application/json; charset=utf-8 + Date: Mon, 14 Oct 2019 16:47:33 GMT + Server: Python/3.7 aiohttp/3.6.1 + + { + "expires_in": 300, + "issued_at": "2019-10-14T16:47:33.107118Z", + "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q" + } + +Use the generated token to access the root again:: + + $ http 'localhost:24816/v2/' --auth-type=jwt --auth="eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q" + + HTTP/1.1 200 OK + Content-Length: 2 + Content-Type: application/json; charset=utf-8 + Date: Mon, 14 Oct 2019 16:50:26 GMT + Docker-Distribution-API-Version: registry/2.0 + Server: Python/3.7 aiohttp/3.6.1 + + {} + +After performing multiple HTTP requests, the root responded with a default value ``{}``. Received +token can be used to access all endpoints within the requested scope too. + +Regular container engines, like docker, or podman, can take advantage of the token authentication. +The authentication is handled by the engines as shown before. + +.. code-block:: bash + + podman pull localhost:24816/foo/bar diff --git a/docs/workflows/host.rst b/docs/workflows/host.rst new file mode 100644 index 00000000..915d0b1a --- /dev/null +++ b/docs/workflows/host.rst @@ -0,0 +1,107 @@ +.. _host: + +Host and Consume a Docker Repository +==================================== + +This section assumes that you have a repository with content in it. To do this, see the +:doc:`sync` documentation. + +Create a Docker Distribution to serve your Repository Version +------------------------------------------------------------- + +Docker Distributions can be used to serve the Docker registry API +containing the content in a repository's latest version or a specified +repository version. + +.. literalinclude:: ../_scripts/distribution.sh + :language: bash + +Response: + +.. code:: + + { + "pulp_created": "2019-09-05T14:29:51.742086Z", + "pulp_href": "/pulp/api/v3/distributions/docker/docker/1b461dac-0839-4049-aa8f-92f8e8f7f034/", + "base_path": "test", + "content_guard": null, + "name": "testing-hello", + "registry_path": "localhost:24816/test", + "repository": "/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/", + "repository_version": null + } + + + +Reference: `Docker Distribution Usage <../restapi.html#tag/distributions>`_ + +Pull and run an image from Pulp +------------------------------- + +Once a distribution is configured to host a repository with Docker +images in it, that content can be consumed by container clients. + +Podman +^^^^^^ + +``$ podman pull localhost:24816/foo`` + +If SSL has not been setup for your Pulp, configure podman to work with the insecure registry: + +Edit the file ``/etc/containers/registries.conf.`` and add:: + + [registries.insecure] + registries = ['localhost:24816'] + +More info: +https://www.projectatomic.io/blog/2018/05/podman-tls/ + +Docker +^^^^^^ + +If SSL has not been setup for your Pulp, configure docker to work with the insecure registry: + +Edit the file ``/etc/docker/daemon.json`` and add:: + + { + "insecure-registries" : ["localhost:24816"] + + } + +More info: +https://docs.docker.com/registry/insecure/#deploy-a-plain-http-registry + +.. literalinclude:: ../_scripts/download_after_sync.sh + :language: bash + +Docker Output:: + + Unable to find image 'localhost:24816/test:latest' locally + Trying to pull repository localhost:24816/test ... + sha256:451ce787d12369c5df2a32c85e5a03d52cbcef6eb3586dd03075f3034f10adcd: Pulling from localhost:24816/test + 1b930d010525: Pull complete + Digest: sha256:451ce787d12369c5df2a32c85e5a03d52cbcef6eb3586dd03075f3034f10adcd + Status: Downloaded newer image for localhost:24816/test:latest + + Hello from Docker! + This message shows that your installation appears to be working correctly. + + To generate this message, Docker took the following steps: + 1. The Docker client contacted the Docker daemon. + 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. + (amd64) + 3. The Docker daemon created a new container from that image which runs the + executable that produces the output you are currently reading. + 4. The Docker daemon streamed that output to the Docker client, which sent it + to your terminal. + + To try something more ambitious, you can run an Ubuntu container with: + $ docker run -it ubuntu bash + + Share images, automate workflows, and more with a free Docker ID: + https://hub.docker.com/ + + For more examples and ideas, visit: + https://docs.docker.com/get-started/ + + diff --git a/docs/workflows/index.rst b/docs/workflows/index.rst new file mode 100644 index 00000000..b7a39ea5 --- /dev/null +++ b/docs/workflows/index.rst @@ -0,0 +1,58 @@ +.. _workflows-index: + +Workflows +========= + +If you have not yet installed the ``pulp_docker`` plugin on your Pulp installation, please follow our +:doc:`../installation`. These documents will assume you have the environment installed and +ready to go. + +Recommended Tools +----------------- + +**httpie**: +The REST API examples here use `httpie `_ to perform the requests. +The ``httpie`` commands below assume that the user executing the commands has a ``.netrc`` file +in the home directory. The ``.netrc`` should have the following configuration: + +.. code-block:: bash + + machine localhost + login admin + password admin + +One should observe that ``httpie`` uses the configuration retrieved from ``.netrc`` by default. +Due to this, a custom Authorization header is always overwritten by the Basic Authorization with +the provided login and password. In order to send HTTP requests which contain JWT Authorization +headers, ensure yourself that the plugin `JWTAuth plugin `_ +was already installed. + +If you configured the ``admin`` user with a different password, adjust the configuration +accordingly. If you prefer to specify the username and password with each request, please see +``httpie`` documentation on how to do that. + +**jq**: +This documentation makes use of the `jq library `_ +to parse the json received from requests, in order to get the unique urls generated +when objects are created. To follow this documentation as-is please install the jq +library with: + +``$ sudo dnf install jq`` + +**environtoment variables** +To make these workflows copy/pastable, we make use of environment variables. The first variable to +set is the hostname and port:: + + $ export BASE_ADDR=http://:24817 + + +Container Workflows +------------------- + +.. toctree:: + :maxdepth: 2 + + sync + host + manage-content + authentication diff --git a/docs/workflows/manage-content.rst b/docs/workflows/manage-content.rst new file mode 100644 index 00000000..e6312034 --- /dev/null +++ b/docs/workflows/manage-content.rst @@ -0,0 +1,341 @@ +.. _content-management: + +Manage Docker Content in a Repository +===================================== + +There are multiple ways that users can manage Docker content in repositories: + + 1. :ref:`Tag` or :ref:`Untag` Manifests in a repository. + 2. Recursively :ref:`add` or :ref:`remove` Docker content. + 3. Copy :ref:`tags` or :ref:`manifests ` from source repository. + +.. warning:: + + Users **can but probably should not not** add and remove Docker + content directly using the `repository version create endpoint + `_. + This endpoint should be reserved for advanced usage and is considered + **unsafe** for Docker content, because it is not recursive and it + allows users to create **corrupted repositories**. + +Each of these workflows kicks off a task, and when the task is complete, +a new repository version will have been created. + +.. _tagging-workflow: + +Tagging +------- + +Images are described by manifests. The procedure of an image tagging is +related to manifests because of that. In pulp, it is required to specify +a digest of a manifest in order to create a tag for the corresponding +image. + +Below is provided an example on how to tag an image within a repository. +First, a digest of an existing manifest is selected. Then, a custom tag is +applied to the corresponding manifest. + +.. literalinclude:: ../_scripts/image_tagging.sh + :language: bash + +A new distribution can be created to include the newly created tag. This +allows clients to pull the image with the applied tag. + +.. literalinclude:: ../_scripts/download_after_tagging.sh + :language: bash + +Each tag has to be unique within a repository to prevent ambiguity. When +a user is trying to tag an image with a same name but with a different +digest, the tag associated with the old manifest is going to be +eliminated in a new repository version. Note that a tagging of same +images with existing names still creates a new repository version. + +Reference: `Docker Tagging Usage <../restapi.html#tag/docker:-tag>`_ + +.. _untagging-workflow: + +Untagging +--------- + +An untagging is an inverse operation to the tagging. To remove a tag +applied to an image, it is required to issue the following calls. + +.. literalinclude:: ../_scripts/image_untagging.sh + :language: bash + +Pulp will create a new repository version which will not contain the +corresponding tag. The removed tag however still persists in a database. +When a client tries to untag an image that was already untagged, a new +repository version is created as well. + +Reference: `Docker Untagging Usage <../restapi.html#tag/docker:-untag>`_ + +.. _recursive-add: + +Recursively Add Content to a Repository +--------------------------------------- + +Any Docker content can be added to a repository version with the +recursive-add endpoint. Here, "recursive" means that the content will be +added, as well as all related content. + + +.. _docker-content-relations: + +Relations: + - Adding a **tag** will also add the tagged manifest and its related + content. + - Adding a **manifest** (manifest list) will also add related + manifests and their related content. + - Adding a **manifest** (not manifest list) will also add related + blobs. + +.. note:: + Because tag names are unique within a repository version, adding a tag + with a duplicate name will first remove the existing tag + (non-recursively). + +Begin by following the :ref:`Synchronize ` workflow to +start with a repository that has some content in it. + +Next create a new repository that we can add content to. + +.. literalinclude:: ../_scripts/destination_repo.sh + :language: bash + +Now we recursively add a tag to the destination repository. + +.. literalinclude:: ../_scripts/recursive_add_tag.sh + :language: bash + +We have added our single tag, as well as the content necessary for that +tag to function correctly when pulled by a client. + +New Repository Version:: + + { + "pulp_created": "2019-09-05T19:04:06.152589Z", + "pulp_href": "/pulp/api/v3/repositories/ce642635-dd9b-423f-82c4-86a150b9f5fe/versions/10/", + "base_version": null, + "content_summary": { + "added": { + "docker.tag": { + "count": 1, + "href": "/pulp/api/v3/content/docker/tags/?repository_version_added=/pulp/api/v3/repositories/ce642635-dd9b-423f-82c4-86a150b9f5fe/versions/10/" + } + }, + "present": { + "docker.blob": { + "count": 20, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version=/pulp/api/v3/repositories/ce642635-dd9b-423f-82c4-86a150b9f5fe/versions/10/" + }, + "docker.manifest": { + "count": 10, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version=/pulp/api/v3/repositories/ce642635-dd9b-423f-82c4-86a150b9f5fe/versions/10/" + }, + "docker.tag": { + "count": 1, + "href": "/pulp/api/v3/content/docker/tags/?repository_version=/pulp/api/v3/repositories/ce642635-dd9b-423f-82c4-86a150b9f5fe/versions/10/" + } + }, + "removed": { + "docker.tag": { + "count": 1, + "href": "/pulp/api/v3/content/docker/tags/?repository_version_removed=/pulp/api/v3/repositories/ce642635-dd9b-423f-82c4-86a150b9f5fe/versions/10/" + } + } + }, + "number": 10 + } + +.. note:: + + Directly adding a manifest that happens to be tagged in another repo + will **not** include its tags. + +Reference: `Docker Recursive Add Usage <../restapi.html#tag/docker:-recursive-add>`_ + +.. _recursive-remove: + +Recursively Remove Content from a Repository +-------------------------------------------- + +Any Docker content can be removed from a repository version with the +recursive-remove endpoint. Recursive remove is symmetrical with +recursive add, meaning that performing a recursive-add and a +recursive-remove back-to-back with the same content will result in the +original content set. If other operations (i.e. tagging) are done between +recursive-add and recursive remove, they can break the symmetry. + +Removing a tag also removes the tagged_manifest and its related content, +which is **new behavior with Pulp 3**. If you just want to remove the +tag, but not the related content, use the :ref:`untagging +workflow`. + + +Recursive remove **does not** remove content that is related to content +that will stay in the repository. For example, if a manifest is tagged, +the manifest cannot be removed from the repository-- instead the tag +should be removed. + +See :ref:`relations` + +Continuing from the :ref:`recursive add workflow`, we can +remove the tag and the related content that is no longer needed. + +.. literalinclude:: ../_scripts/recursive_remove_tag.sh + :language: bash + +Now we can see that the tag and related content that was added has now +been removed, resulting in an empty repository. + +New Repository Version:: + + { + "pulp_created": "2019-09-10T13:25:44.078017Z", + "pulp_href": "/pulp/api/v3/repositories/c2f67416-7200-4dcc-9868-f320431aae20/versions/2/", + "base_version": null, + "content_summary": { + "added": {}, + "present": {}, + "removed": { + "docker.blob": { + "count": 20, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version_removed=/pulp/api/v3/repositories/c2f67416-7200-4dcc-9868-f320431aae20/versions/2/" + }, + "docker.manifest": { + "count": 10, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version_removed=/pulp/api/v3/repositories/c2f67416-7200-4dcc-9868-f320431aae20/versions/2/" + }, + "docker.tag": { + "count": 1, + "href": "/pulp/api/v3/content/docker/tags/?repository_version_removed=/pulp/api/v3/repositories/c2f67416-7200-4dcc-9868-f320431aae20/versions/2/" + } + } + }, + "number": 2 + } + +Reference: `Docker Recursive Remove Usage <../restapi.html#tag/docker:-recursive-remove>`_ + +.. _tag-copy: + +Recursively Copy Tags from a Source Repository +---------------------------------------------- + +Tags in one repository can be copied to another repository using the tag +copy endpoint. + +When no names are specified, all tags are recursively copied. If names are +specified, only the matching tags are recursively copied. + +If tag names being copied already exist in the destination repository, +the conflicting tags are removed from the destination repository and the +new tags are added. This action is not recursive, no manifests or blobs +are removed. + +Again we start with a new destination repository. + +.. literalinclude:: ../_scripts/destination_repo.sh + :language: bash + +With copy (contrasted to recursive add) we do not need to retrieve the +href of the tag. Rather, we can specify the tag by source repository and +name. + +.. literalinclude:: ../_scripts/tag_copy.sh + :language: bash + +New Repository Version:: + + { + "pulp_created": "2019-09-10T13:42:12.572859Z", + "pulp_href": "/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/", + "base_version": null, + "content_summary": { + "added": { + "docker.blob": { + "count": 20, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version_added=/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/" + }, + "docker.manifest": { + "count": 10, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version_added=/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/" + }, + "docker.tag": { + "count": 1, + "href": "/pulp/api/v3/content/docker/tags/?repository_version_added=/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/" + } + }, + "present": { + "docker.blob": { + "count": 20, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version=/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/" + }, + "docker.manifest": { + "count": 10, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version=/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/" + }, + "docker.tag": { + "count": 1, + "href": "/pulp/api/v3/content/docker/tags/?repository_version=/pulp/api/v3/repositories/2b1c6d76-c369-4f31-8eb8-9d5d92bb2346/versions/1/" + } + }, + "removed": {} + }, + "number": 1 + } + +Reference: `Docker Copy Tags Usage <../restapi.html#operation/docker_tags_copy_create>`_ + +.. _manifest-copy: + +Recursively Copy Manifests from a Source Repository +--------------------------------------------------- + +Manifests in one repository can be copied to another repository using +the manifest copy endpoint. + +If digests are specified, only the manifests (and their recursively +related content) will be added. + +If media_types are specified, only manifests matching that media type +(and their recursively related content) will be added. This allows users +to copy only manifest lists, for example. + +.. literalinclude:: ../_scripts/manifest_copy.sh + :language: bash + +New Repository Version:: + + { + "pulp_created": "2019-09-20T13:53:04.907351Z", + "pulp_href": "/pulp/api/v3/repositories/70450dfb-ae46-4061-84e3-97eb71cf9414/versions/2/", + "base_version": null, + "content_summary": { + "added": { + "docker.blob": { + "count": 31, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version_added=/pulp/api/v3/repositories/70450dfb-ae46-4061-84e3-97eb71cf9414/versions/2/" + }, + "docker.manifest": { + "count": 21, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version_added=/pulp/api/v3/repositories/70450dfb-ae46-4061-84e3-97eb71cf9414/versions/2/" + } + }, + "present": { + "docker.blob": { + "count": 31, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version=/pulp/api/v3/repositories/70450dfb-ae46-4061-84e3-97eb71cf9414/versions/2/" + }, + "docker.manifest": { + "count": 21, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version=/pulp/api/v3/repositories/70450dfb-ae46-4061-84e3-97eb71cf9414/versions/2/" + } + }, + "removed": {} + }, + "number": 2 + } + +Reference: `Docker Copy Manifests Usage <../restapi.html#operation/docker_manifests_copy_create>`_ diff --git a/docs/workflows/sync.rst b/docs/workflows/sync.rst new file mode 100644 index 00000000..b36be9db --- /dev/null +++ b/docs/workflows/sync.rst @@ -0,0 +1,120 @@ +.. _sync-workflow: + +Synchronize a Repository +======================== + +Users can populate their repositories with content from an external source like Docker Hub by syncing +their repository. + +Create a Repository +------------------- + +.. literalinclude:: ../_scripts/repo.sh + :language: bash + +Repository GET Response:: + + { + "pulp_created": "2019-09-05T14:29:43.424822Z", + "pulp_href": "/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/", + "latest_version_href": null, + "versions_href": "/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/", + "description": null, + "name": "codzo" + } + +Reference (pulpcore): `Repository API Usage +`_ + +.. _create-remote: + +Create a Remote +--------------- + +Creating a remote object informs Pulp about an external content source. In this case, we will be +using Docker Hub, but ``pulp-docker`` remotes can be anything that implements the registry API, +including `quay`, `google container registry`, or even another instance of Pulp. + +.. literalinclude:: ../_scripts/remote.sh + :language: bash + +Remote GET Response:: + + { + "pulp_created": "2019-09-05T14:29:44.267406Z", + "pulp_href": "/pulp/api/v3/remotes/docker/docker/1cc699b7-24fd-4944-bde7-86aed8ac12fa/", + "pulp_last_updated": "2019-09-05T14:29:44.267428Z", + "download_concurrency": 20, + "name": "my-hello-repo", + "policy": "immediate", + "proxy_url": null, + "ssl_ca_certificate": null, + "ssl_client_certificate": null, + "ssl_client_key": null, + "ssl_validation": true, + "upstream_name": "library/hello-world", + "url": "https://registry-1.docker.io", + "whitelist_tags": null + } + + +Reference: `Docker Remote Usage <../restapi.html#tag/remotes>`_ + +Sync repository using a Remote +------------------------------ + +Use the remote object to kick off a synchronize task by specifying the repository to +sync with. You are telling pulp to fetch content from the remote and add to the repository. + +.. literalinclude:: ../_scripts/sync.sh + :language: bash + +Reference: `Docker Sync Usage <../restapi.html#operation/remotes_docker_docker_sync>`_ + + +.. _versioned-repo-created: + +Repository Version GET Response (when complete): + +.. code:: json + + { + "pulp_created": "2019-09-05T14:29:45.563089Z", + "pulp_href": "/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/", + "base_version": null, + "content_summary": { + "added": { + "docker.blob": { + "count": 31, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version_added=/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/" + }, + "docker.manifest": { + "count": 21, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version_added=/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/" + }, + "docker.tag": { + "count": 8, + "href": "/pulp/api/v3/content/docker/tags/?repository_version_added=/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/" + } + }, + "present": { + "docker.blob": { + "count": 31, + "href": "/pulp/api/v3/content/docker/blobs/?repository_version=/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/" + }, + "docker.manifest": { + "count": 21, + "href": "/pulp/api/v3/content/docker/manifests/?repository_version=/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/" + }, + "docker.tag": { + "count": 8, + "href": "/pulp/api/v3/content/docker/tags/?repository_version=/pulp/api/v3/repositories/fcf03266-f0e4-4497-8434-0fe9d94c8053/versions/1/" + } + }, + "removed": {} + }, + "number": 1 + } + +Reference (pulpcore): `Repository Version API Usage +`_ diff --git a/extensions_admin/pulp_docker/__init__.py b/extensions_admin/pulp_docker/__init__.py deleted file mode 100644 index 3ad9513f..00000000 --- a/extensions_admin/pulp_docker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/extensions_admin/pulp_docker/extensions/__init__.py b/extensions_admin/pulp_docker/extensions/__init__.py deleted file mode 100644 index 3ad9513f..00000000 --- a/extensions_admin/pulp_docker/extensions/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/extensions_admin/pulp_docker/extensions/admin/cudl.py b/extensions_admin/pulp_docker/extensions/admin/cudl.py deleted file mode 100644 index 00c0f409..00000000 --- a/extensions_admin/pulp_docker/extensions/admin/cudl.py +++ /dev/null @@ -1,242 +0,0 @@ -from gettext import gettext as _ - -from pulp.client import arg_utils, parsers -from pulp.client.commands.options import OPTION_REPO_ID -from pulp.client.commands.repo.cudl import CreateAndConfigureRepositoryCommand -from pulp.client.commands.repo.cudl import UpdateRepositoryCommand -from pulp.client.commands.repo.importer_config import ImporterConfigMixin -from pulp.common.constants import REPO_NOTE_TYPE_KEY -from pulp.client.extensions.extensions import PulpCliOption - -from pulp_docker.common import constants, tags -from pulp_docker.extensions.admin import parsers as docker_parsers - - -d = _('if "true", on each successful sync the repository will automatically be ' - 'published; if "false" content will only be available after manually publishing ' - 'the repository; defaults to "true"') -OPT_AUTO_PUBLISH = PulpCliOption('--auto-publish', d, required=False, - parse_func=parsers.parse_boolean) - -d = _('The URL that will be used when generating the redirect map for connecting the docker ' - 'API to the location the content is stored. ' - 'The value defaults to https:///pulp/docker/.') -OPT_REDIRECT_URL = PulpCliOption('--redirect-url', d, required=False) - -d = _('the name that will be used for this repository in the Docker registry. If not specified, ' - 'the repo id will be used') -OPT_REPO_REGISTRY_ID = PulpCliOption('--repo-registry-id', d, required=False) - -d = _('if "true" requests for this repo will be checked for an entitlement certificate authorizing ' - 'the server url for this repository; if "false" no authorization checking will be done.') -OPT_PROTECTED = PulpCliOption('--protected', d, required=False, parse_func=parsers.parse_boolean) - -d = _('Tag a particular image in the repository. The format of the parameter is ' - '":"; for example: "latest:abc123"') -OPTION_TAG = PulpCliOption('--tag', d, required=False, allow_multiple=True, - parse_func=docker_parsers.parse_colon_separated) - -d = _('Remove the specified tag from the repository. This only removes the tag; the underlying ' - 'image will remain in the repository.') -OPTION_REMOVE_TAG = PulpCliOption('--remove-tag', d, required=False, allow_multiple=True) - -d = _('name of the upstream repository') -OPT_UPSTREAM_NAME = PulpCliOption('--upstream-name', d, required=False) - -DESC_FEED = _('URL for the upstream docker index, not including repo name') - - -class CreateDockerRepositoryCommand(CreateAndConfigureRepositoryCommand, ImporterConfigMixin): - default_notes = {REPO_NOTE_TYPE_KEY: constants.REPO_NOTE_DOCKER} - IMPORTER_TYPE_ID = constants.IMPORTER_TYPE_ID - - def __init__(self, context): - CreateAndConfigureRepositoryCommand.__init__(self, context) - ImporterConfigMixin.__init__(self, include_ssl=False, include_sync=True, - include_unit_policy=False) - self.add_option(OPT_AUTO_PUBLISH) - self.add_option(OPT_REDIRECT_URL) - self.add_option(OPT_PROTECTED) - self.add_option(OPT_REPO_REGISTRY_ID) - self.sync_group.add_option(OPT_UPSTREAM_NAME) - self.options_bundle.opt_feed.description = DESC_FEED - - def _describe_distributors(self, user_input): - """ - Subclasses should override this to provide whatever option parsing - is needed to create distributor configs. - - :param user_input: dictionary of data passed in by okaara - :type user_inpus: dict - - :return: list of dict containing distributor_type_id, - repo_plugin_config, auto_publish, and distributor_id (the same - that would be passed to the RepoDistributorAPI.create call). - :rtype: list of dict - """ - config = {} - value = user_input.pop(OPT_PROTECTED.keyword, None) - if value is not None: - config[constants.CONFIG_KEY_PROTECTED] = value - - value = user_input.pop(OPT_REDIRECT_URL.keyword, None) - if value is not None: - config[constants.CONFIG_KEY_REDIRECT_URL] = value - - value = user_input.pop(OPT_REPO_REGISTRY_ID.keyword, None) - if value is not None: - config[constants.CONFIG_KEY_REPO_REGISTRY_ID] = value - - auto_publish = user_input.get('auto-publish', True) - data = [ - dict(distributor_type_id=constants.DISTRIBUTOR_WEB_TYPE_ID, - distributor_config=config, - auto_publish=auto_publish, - distributor_id=constants.CLI_WEB_DISTRIBUTOR_ID), - dict(distributor_type_id=constants.DISTRIBUTOR_EXPORT_TYPE_ID, - distributor_config=config, - auto_publish=False, distributor_id=constants.CLI_EXPORT_DISTRIBUTOR_ID) - ] - - return data - - def _parse_importer_config(self, user_input): - """ - Subclasses should override this to provide whatever option parsing - is needed to create an importer config. - - :param user_input: dictionary of data passed in by okaara - :type user_inpus: dict - - :return: importer config - :rtype: dict - """ - config = self.parse_user_input(user_input) - - name = user_input.pop(OPT_UPSTREAM_NAME.keyword) - if name is not None: - config[constants.CONFIG_KEY_UPSTREAM_NAME] = name - - return config - - -class UpdateDockerRepositoryCommand(UpdateRepositoryCommand, ImporterConfigMixin): - - def __init__(self, context): - UpdateRepositoryCommand.__init__(self, context) - ImporterConfigMixin.__init__(self, include_ssl=False, include_sync=True, - include_unit_policy=False) - self.add_option(OPTION_TAG) - self.add_option(OPTION_REMOVE_TAG) - self.add_option(OPT_AUTO_PUBLISH) - self.add_option(OPT_REDIRECT_URL) - self.add_option(OPT_PROTECTED) - self.add_option(OPT_REPO_REGISTRY_ID) - self.sync_group.add_option(OPT_UPSTREAM_NAME) - self.options_bundle.opt_feed.description = DESC_FEED - - def run(self, **kwargs): - arg_utils.convert_removed_options(kwargs) - - importer_config = self.parse_user_input(kwargs) - - name = kwargs.pop(OPT_UPSTREAM_NAME.keyword, None) - if name is not None: - importer_config[constants.CONFIG_KEY_UPSTREAM_NAME] = name - - if importer_config: - kwargs['importer_config'] = importer_config - - # Update distributor configuration - web_config = {} - export_config = {} - value = kwargs.pop(OPT_PROTECTED.keyword, None) - if value is not None: - web_config[constants.CONFIG_KEY_PROTECTED] = value - - value = kwargs.pop(OPT_REDIRECT_URL.keyword, None) - if value is not None: - web_config[constants.CONFIG_KEY_REDIRECT_URL] = value - export_config[constants.CONFIG_KEY_REDIRECT_URL] = value - - value = kwargs.pop(OPT_REPO_REGISTRY_ID.keyword, None) - if value is not None: - web_config[constants.CONFIG_KEY_REPO_REGISTRY_ID] = value - export_config[constants.CONFIG_KEY_REPO_REGISTRY_ID] = value - - value = kwargs.pop(OPT_AUTO_PUBLISH.keyword, None) - if value is not None: - web_config['auto_publish'] = value - - if web_config or export_config: - kwargs['distributor_configs'] = {} - - if web_config: - kwargs['distributor_configs'][constants.CLI_WEB_DISTRIBUTOR_ID] = web_config - - if export_config: - kwargs['distributor_configs'][constants.CLI_EXPORT_DISTRIBUTOR_ID] = export_config - - # Update Tags - repo_id = kwargs.get(OPTION_REPO_ID.keyword) - response = self.context.server.repo.repository(repo_id).response_body - scratchpad = response.get(u'scratchpad', {}) - image_tags = scratchpad.get(u'tags', []) - - user_tags = kwargs.get(OPTION_TAG.keyword) - if user_tags: - user_tags = kwargs.pop(OPTION_TAG.keyword) - for tag, image_id in user_tags: - if len(image_id) < 6: - msg = _('The image id, (%s), must be at least 6 characters.') - self.prompt.render_failure_message(msg % image_id) - return - - # Ensure the specified images exist in the repo - images_requested = set([image_id for tag, image_id in user_tags]) - images = ['^%s' % image_id for image_id in images_requested] - image_regex = '|'.join(images) - search_criteria = { - 'type_ids': constants.IMAGE_TYPE_ID, - 'match': [['image_id', image_regex]], - 'fields': ['image_id'] - } - - response = self.context.server.repo_unit.search(repo_id, **search_criteria).\ - response_body - if len(response) != len(images): - images_found = set([x[u'metadata'][u'image_id'] for x in response]) - missing_images = images_requested.difference(images_found) - msg = _('Unable to create tag in repository. The following image(s) do not ' - 'exist in the repository: %s.') - self.prompt.render_failure_message(msg % ', '.join(missing_images)) - return - - # Get the full image id from the returned values and save in tags_to_update dictionary - tags_to_update = {} - for image in response: - found_image_id = image[u'metadata'][u'image_id'] - for tag, image_id in user_tags: - if found_image_id.startswith(image_id): - tags_to_update[tag] = found_image_id - - # Create a list of tag dictionaries that can be saved on the repo scratchpad - # using the original tags and new tags specified by the user - image_tags = tags.generate_updated_tags(scratchpad, tags_to_update) - scratchpad[u'tags'] = image_tags - kwargs[u'scratchpad'] = scratchpad - - remove_tags = kwargs.get(OPTION_REMOVE_TAG.keyword) - if remove_tags: - kwargs.pop(OPTION_REMOVE_TAG.keyword) - for tag in remove_tags: - # For each tag in remove_tags, remove the respective tag dictionary - # for matching tag. - for image_tag in image_tags[:]: - if tag == image_tag[constants.IMAGE_TAG_KEY]: - image_tags.remove(image_tag) - - scratchpad[u'tags'] = image_tags - kwargs[u'scratchpad'] = scratchpad - - super(UpdateDockerRepositoryCommand, self).run(**kwargs) diff --git a/extensions_admin/pulp_docker/extensions/admin/images.py b/extensions_admin/pulp_docker/extensions/admin/images.py deleted file mode 100644 index 3fbe3f1b..00000000 --- a/extensions_admin/pulp_docker/extensions/admin/images.py +++ /dev/null @@ -1,110 +0,0 @@ -from gettext import gettext as _ - -from pulp.client.commands import options -from pulp.client.commands.criteria import DisplayUnitAssociationsCommand -from pulp.client.commands.unit import UnitCopyCommand, UnitRemoveCommand - -from pulp_docker.common import constants - - -DESC_COPY = _('copies images from one repository into another') -DESC_REMOVE = _('remove images from a repository') -DESC_SEARCH = _('search for images in a repository') - -MODULE_ID_TEMPLATE = '%(image_id)s' - - -def get_formatter_for_type(type_id): - """ - Return a method that takes one argument (a unit) and formats a short string - to be used as the output for the unit_remove command - - :param type_id: The type of the unit for which a formatter is needed - :type type_id: str - :raises ValueError: if the method does not recognize the type_id - """ - - if type_id != constants.IMAGE_TYPE_ID: - raise ValueError(_("The docker image formatter can not process %s units.") % type_id) - - return lambda x: MODULE_ID_TEMPLATE % x - - -class ImageCopyCommand(UnitCopyCommand): - - def __init__(self, context, name='copy', description=DESC_COPY): - super(ImageCopyCommand, self).__init__(context, name=name, description=description, - method=self.run, type_id=constants.IMAGE_TYPE_ID) - - @staticmethod - def get_formatter_for_type(type_id): - """ - Returns a method that can be used to format the unit key of a docker image - for display purposes - - :param type_id: the type_id of the unit key to get a formatter for - :type type_id: str - :return: function - """ - return get_formatter_for_type(type_id) - - -class ImageRemoveCommand(UnitRemoveCommand): - """ - Class for executing unit remove commands for docker image units - """ - - def __init__(self, context, name='remove', description=DESC_REMOVE): - UnitRemoveCommand.__init__(self, context, name=name, description=description, - type_id=constants.IMAGE_TYPE_ID) - - @staticmethod - def get_formatter_for_type(type_id): - """ - Returns a method that can be used to format the unit key of a puppet_module - for display purposes - - :param type_id: the type_id of the unit key to get a formatter for - :type type_id: str - :return: function - """ - return get_formatter_for_type(type_id) - - -class ImageSearchCommand(DisplayUnitAssociationsCommand): - def __init__(self, context): - super(ImageSearchCommand, self).__init__(self.run, name='images', description=DESC_SEARCH) - self.context = context - self.prompt = context.prompt - - def run(self, **kwargs): - """ - Print a list of all the images matching the search parameters in kwargs - - :param kwargs: the search parameters for finding docker images - :type kwargs: dict - """ - # Get the list of images - repo_id = kwargs.pop(options.OPTION_REPO_ID.keyword) - kwargs['type_ids'] = [constants.IMAGE_TYPE_ID] - images = self.context.server.repo_unit.search(repo_id, **kwargs).response_body - - # Get the list of tags for the repo - response = self.context.server.repo.repository(repo_id).response_body - scratchpad = response.get(u'scratchpad', {}) - tags = scratchpad.get(u'tags', []) - image_tags = {} - - # create a dictionary to map images to a list of tags - for tag_dict in tags: - tag = tag_dict[constants.IMAGE_TAG_KEY] - image_id = tag_dict[constants.IMAGE_ID_KEY] - image_tags.setdefault(image_id, []).append(tag) - - # Add the tag info to the images list - for image in images: - image_id = image[u'metadata'][u'image_id'] - if image_id in image_tags: - image[u'metadata'][u'tags'] = image_tags.get(image_id) - - self.prompt.render_document_list(images) diff --git a/extensions_admin/pulp_docker/extensions/admin/parsers.py b/extensions_admin/pulp_docker/extensions/admin/parsers.py deleted file mode 100644 index b0533a32..00000000 --- a/extensions_admin/pulp_docker/extensions/admin/parsers.py +++ /dev/null @@ -1,26 +0,0 @@ -from gettext import gettext as _ - - -def parse_colon_separated(input_value): - """ - Parses a colon separated key value pair. - The given value will actually be a list of values regardless - of whether or not the user specified multiple notes. - - :param input_value: list of user entered values or empty list if unspecified - :type input_value: list - :return: list of tuples (tag_name, image_hash) - :rtype: list of (str, str) - :raises ValueError: if the value can not be parsed - """ - if input_value: - ret = [x.rsplit(':', 1) for x in input_value] - for value in ret: - msg = _('Unable to parse %s, value should be in the format "aaa:bbb"') - if len(value) != 2: - raise ValueError(msg % value) - elif not len(value[0]) or not len(value[1]): - raise ValueError(msg % value) - return ret - else: - return [] diff --git a/extensions_admin/pulp_docker/extensions/admin/pulp_cli.py b/extensions_admin/pulp_docker/extensions/admin/pulp_cli.py deleted file mode 100644 index 9eaf681f..00000000 --- a/extensions_admin/pulp_docker/extensions/admin/pulp_cli.py +++ /dev/null @@ -1,154 +0,0 @@ -from gettext import gettext as _ - -from pulp.client.commands.repo import cudl, sync_publish, status -from pulp.client.extensions.decorator import priority -from pulp.client.extensions.extensions import PulpCliOption - -from pulp_docker.common import constants -from pulp_docker.extensions.admin.cudl import CreateDockerRepositoryCommand -from pulp_docker.extensions.admin.cudl import UpdateDockerRepositoryCommand -from pulp_docker.extensions.admin.images import ImageCopyCommand -from pulp_docker.extensions.admin.images import ImageRemoveCommand -from pulp_docker.extensions.admin.images import ImageSearchCommand -from pulp_docker.extensions.admin.upload import UploadDockerImageCommand -from pulp_docker.extensions.admin.repo_list import ListDockerRepositoriesCommand - - -SECTION_ROOT = 'docker' -DESC_ROOT = _('manage docker images') - -SECTION_REPO = 'repo' -DESC_REPO = _('repository lifecycle commands') - -SECTION_UPLOADS = 'uploads' -DESC_UPLOADS = _('upload docker images into a repository') - -SECTION_SYNC = 'sync' -DESC_SYNC = _('sync a docker repository from an upstream index') - -SECTION_PUBLISH = 'publish' -DESC_PUBLISH = _('publish a docker repository') - -SECTION_EXPORT = 'export' -DESC_EXPORT = _('export a docker repository') -DESC_EXPORT_RUN = _('triggers an immediate export of a repository to a tar file') -DESC_EXPORT_FILE = _('the full path for an export file; if specified, the repository will be ' - 'exported as a tar file to the given file on the server. ' - 'The web server\'s user must have the permission to write the file specified.') - -OPTION_EXPORT_FILE = PulpCliOption('--export-file', DESC_EXPORT_FILE, required=False) - - -@priority() -def initialize(context): - """ - create the docker CLI section and add it to the root - - :type context: pulp.client.extensions.core.ClientContext - """ - root_section = context.cli.create_section(SECTION_ROOT, DESC_ROOT) - repo_section = add_repo_section(context, root_section) - add_upload_section(context, repo_section) - add_sync_section(context, repo_section) - add_publish_section(context, repo_section) - add_export_section(context, repo_section) - - -def add_upload_section(context, parent_section): - """ - add an upload section to the docker section - - :type context: pulp.client.extensions.core.ClientContext - :param parent_section: section of the CLI to which the upload section - should be added - :type parent_section: pulp.client.extensions.extensions.PulpCliSection - """ - upload_section = parent_section.create_subsection(SECTION_UPLOADS, DESC_UPLOADS) - - upload_section.add_command(UploadDockerImageCommand(context)) - - return upload_section - - -def add_repo_section(context, parent_section): - """ - add a repo section to the docker section - - :type context: pulp.client.extensions.core.ClientContext - :param parent_section: section of the CLI to which the repo section - should be added - :type parent_section: pulp.client.extensions.extensions.PulpCliSection - """ - repo_section = parent_section.create_subsection(SECTION_REPO, DESC_REPO) - - repo_section.add_command(CreateDockerRepositoryCommand(context)) - repo_section.add_command(cudl.DeleteRepositoryCommand(context)) - repo_section.add_command(UpdateDockerRepositoryCommand(context)) - repo_section.add_command(ImageRemoveCommand(context)) - repo_section.add_command(ImageCopyCommand(context)) - repo_section.add_command(ImageSearchCommand(context)) - repo_section.add_command(ListDockerRepositoriesCommand(context)) - - return repo_section - - -def add_sync_section(context, parent_section): - """ - add a sync section - - :param context: pulp context - :type context: pulp.client.extensions.core.ClientContext - :param parent_section: section of the CLI to which the upload section - should be added - :type parent_section: pulp.client.extensions.extensions.PulpCliSection - :return: populated section - :rtype: PulpCliSection - """ - renderer = status.PublishStepStatusRenderer(context) - - sync_section = parent_section.create_subsection(SECTION_SYNC, DESC_SYNC) - sync_section.add_command(sync_publish.RunSyncRepositoryCommand(context, renderer)) - - return sync_section - - -def add_publish_section(context, parent_section): - """ - add a publish section to the repo section - - :type context: pulp.client.extensions.core.ClientContext - :param parent_section: section of the CLI to which the repo section should be added - :type parent_section: pulp.client.extensions.extensions.PulpCliSection - """ - section = parent_section.create_subsection(SECTION_PUBLISH, DESC_PUBLISH) - - renderer = status.PublishStepStatusRenderer(context) - section.add_command( - sync_publish.RunPublishRepositoryCommand(context, - renderer, - constants.CLI_WEB_DISTRIBUTOR_ID)) - section.add_command( - sync_publish.PublishStatusCommand(context, renderer)) - - return section - - -def add_export_section(context, parent_section): - """ - add a export section to the parent section - - :type context: pulp.client.extensions.core.ClientContext - :param parent_section: section of the CLI to which the export section should be added - :type parent_section: pulp.client.extensions.extensions.PulpCliSection - """ - section = parent_section.create_subsection(SECTION_EXPORT, DESC_EXPORT) - section.add_command( - sync_publish.RunPublishRepositoryCommand(context=context, - renderer=status.PublishStepStatusRenderer(context), - distributor_id=constants.CLI_EXPORT_DISTRIBUTOR_ID, - description=DESC_EXPORT_RUN, - override_config_options=[OPTION_EXPORT_FILE])) - section.add_command( - sync_publish.PublishStatusCommand(context, status.PublishStepStatusRenderer(context))) - - return section diff --git a/extensions_admin/pulp_docker/extensions/admin/repo_list.py b/extensions_admin/pulp_docker/extensions/admin/repo_list.py deleted file mode 100644 index 5b7999c1..00000000 --- a/extensions_admin/pulp_docker/extensions/admin/repo_list.py +++ /dev/null @@ -1,87 +0,0 @@ -from gettext import gettext as _ - -from pulp.client.commands.repo.cudl import ListRepositoriesCommand -from pulp.common import constants as pulp_constants - -from pulp_docker.common import constants - - -class ListDockerRepositoriesCommand(ListRepositoriesCommand): - - def __init__(self, context): - repos_title = _('Docker Repositories') - super(ListDockerRepositoriesCommand, self).__init__(context, repos_title=repos_title) - - # Both get_repositories and get_other_repositories will act on the full - # list of repositories. Lazy cache the data here since both will be - # called in succession, saving the round trip to the server. - self.all_repos_cache = None - - def get_repositories(self, query_params, **kwargs): - """ - Get a list of all the docker repositories that match the specified query params - - :param query_params: query parameters for refining the list of repositories - :type query_params: dict - :param kwargs: Any additional parameters passed into the repo list command - :type kwargs: dict - :return: List of docker repositories - :rtype: list of dict - """ - all_repos = self._all_repos(query_params, **kwargs) - - docker_repos = [] - for repo in all_repos: - notes = repo['notes'] - if pulp_constants.REPO_NOTE_TYPE_KEY in notes \ - and notes[pulp_constants.REPO_NOTE_TYPE_KEY] == constants.REPO_NOTE_DOCKER: - docker_repos.append(repo) - - # There isn't really anything compelling in the exporter distributor - # to display to the user, so remove it entirely. - for r in docker_repos: - if 'distributors' in r: - r['distributors'] = \ - [x for x in r['distributors'] if x['id'] == constants.CLI_EXPORT_DISTRIBUTOR_ID] - - return docker_repos - - def get_other_repositories(self, query_params, **kwargs): - """ - Get a list of all the non docker repositories that match the specified query params - - :param query_params: query parameters for refining the list of repositories - :type query_params: dict - :param kwargs: Any additional parameters passed into the repo list command - :type kwargs: dict - :return: List of non repositories - :rtype: list of dict - """ - - all_repos = self._all_repos(query_params, **kwargs) - - non_docker_repos = [] - for repo in all_repos: - notes = repo['notes'] - if notes.get(pulp_constants.REPO_NOTE_TYPE_KEY, None) != constants.REPO_NOTE_DOCKER: - non_docker_repos.append(repo) - - return non_docker_repos - - def _all_repos(self, query_params, **kwargs): - """ - get all the repositories associated with a repo that match a set of query parameters - - :param query_params: query parameters for refining the list of repositories - :type query_params: dict - :param kwargs: Any additional parameters passed into the repo list command - :type kwargs: dict - :return: list of repositories - :rtype: list of dict - """ - - # This is safe from any issues with concurrency due to how the CLI works - if self.all_repos_cache is None: - self.all_repos_cache = self.context.server.repo.repositories(query_params).response_body - - return self.all_repos_cache diff --git a/extensions_admin/pulp_docker/extensions/admin/upload.py b/extensions_admin/pulp_docker/extensions/admin/upload.py deleted file mode 100644 index 3dff6b0b..00000000 --- a/extensions_admin/pulp_docker/extensions/admin/upload.py +++ /dev/null @@ -1,66 +0,0 @@ -from gettext import gettext as _ - - -from pulp.client.commands.repo.upload import UploadCommand -from pulp.client.extensions.extensions import PulpCliOption - -from pulp_docker.common import constants - - -d = _('image id of an ancestor image that should not be added to the repository. ' - 'The masked ancestor and any ancestors of that image will be skipped from importing into ' - 'the repository.') -OPT_MASK_ANCESTOR_ID = PulpCliOption('--mask-id', d, aliases=['-m'], required=False) - - -class UploadDockerImageCommand(UploadCommand): - - def __init__(self, context): - super(UploadDockerImageCommand, self).__init__(context) - self.add_option(OPT_MASK_ANCESTOR_ID) - - def determine_type_id(self, filename, **kwargs): - """ - We only support one content type, so this always returns that. - - :return: ID of the type of file being uploaded - :rtype: str - """ - return constants.IMAGE_TYPE_ID - - def generate_unit_key_and_metadata(self, filename, **kwargs): - """ - Returns unit key and metadata as empty dictionaries. This is appropriate - in this case, since docker image consists of multiple layers each having - it's own unit key and each layer is imported separately into the server database. - - :param filename: full path to the file being uploaded - :type filename: str, None - - :param kwargs: arguments passed into the upload call by the user - :type kwargs: dict - - :return: tuple of unit key and metadata to upload for the file - :rtype: tuple - """ - unit_key = {} - metadata = {} - - return unit_key, metadata - - def generate_override_config(self, **kwargs): - """ - Generate an override config value to the upload command. - - :param kwargs: parsed from the user input - :type kwargs: dict - - :return: override config generated from the user input - :rtype: dict - """ - override_config = {} - - if OPT_MASK_ANCESTOR_ID.keyword in kwargs: - override_config[constants.CONFIG_KEY_MASK_ID] = kwargs[OPT_MASK_ANCESTOR_ID.keyword] - - return override_config diff --git a/extensions_admin/setup.py b/extensions_admin/setup.py deleted file mode 100644 index 0a96cc14..00000000 --- a/extensions_admin/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_docker_extensions_admin', - version='0.1.0', - packages=find_packages(), - url='http://www.pulpproject.org', - license='GPLv2+', - author='Pulp Team', - author_email='pulp-list@redhat.com', - description='pulp-admin extensions for docker image support', - entry_points={ - 'pulp.extensions.admin': [ - 'repo_admin = pulp_docker.extensions.admin.pulp_cli:initialize', - ] - } -) diff --git a/extensions_admin/test/data/busyboxlight.tar b/extensions_admin/test/data/busyboxlight.tar deleted file mode 100644 index 3ad09df1..00000000 Binary files a/extensions_admin/test/data/busyboxlight.tar and /dev/null differ diff --git a/extensions_admin/test/unit/extensions/__init__.py b/extensions_admin/test/unit/extensions/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/extensions_admin/test/unit/extensions/admin/__init__.py b/extensions_admin/test/unit/extensions/admin/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/extensions_admin/test/unit/extensions/admin/data.py b/extensions_admin/test/unit/extensions/admin/data.py deleted file mode 100644 index 3f2a7297..00000000 --- a/extensions_admin/test/unit/extensions/admin/data.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - - -busybox_tar_path = os.path.join(os.path.dirname(__file__), '../../../data/busyboxlight.tar') - -# these are in correct ancestry order -busybox_ids = ( - '769b9341d937a3dba9e460f664b4f183a6cecdd62b337220a28b3deb50ee0a02', - '48e5f45168b97799ad0aafb7e2fef9fac57b5f16f6db7f67ba2000eb947637eb', - 'bf747efa0e2fa9f7c691588ce3938944c75607a7bb5e757f7369f86904d97c78', - '511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158', -) diff --git a/extensions_admin/test/unit/extensions/admin/test_cudl.py b/extensions_admin/test/unit/extensions/admin/test_cudl.py deleted file mode 100644 index b56c79cb..00000000 --- a/extensions_admin/test/unit/extensions/admin/test_cudl.py +++ /dev/null @@ -1,182 +0,0 @@ -import unittest - -from mock import Mock -from pulp.common.constants import REPO_NOTE_TYPE_KEY -from pulp.devel.unit.util import compare_dict - -from pulp_docker.common import constants -from pulp_docker.extensions.admin import cudl - - -class TestCreateDockerRepositoryCommand(unittest.TestCase): - def test_default_notes(self): - # make sure this value is set and is correct - self.assertEqual(cudl.CreateDockerRepositoryCommand.default_notes.get(REPO_NOTE_TYPE_KEY), - constants.REPO_NOTE_DOCKER) - - def test_importer_id(self): - # this value is required to be set, so just make sure it's correct - self.assertEqual(cudl.CreateDockerRepositoryCommand.IMPORTER_TYPE_ID, - constants.IMPORTER_TYPE_ID) - - def test_describe_distributors(self): - command = cudl.CreateDockerRepositoryCommand(Mock()) - user_input = {'redirect-url': 'foo', - 'protected': False, - 'repo-registry-id': 'bar'} - result = command._describe_distributors(user_input) - target_result = { - 'distributor_type_id': constants.DISTRIBUTOR_WEB_TYPE_ID, - 'distributor_config': { - 'redirect-url': 'foo', 'repo-registry-id': 'bar', 'protected': False}, - 'auto_publish': True, - 'distributor_id': constants.CLI_WEB_DISTRIBUTOR_ID - } - compare_dict(result[0], target_result) - - def test_describe_distributors_override_auto_publish(self): - command = cudl.CreateDockerRepositoryCommand(Mock()) - user_input = { - 'auto-publish': False - } - result = command._describe_distributors(user_input) - self.assertEquals(result[0]["auto_publish"], False) - - def test_parse_importer_config(self): - command = cudl.CreateDockerRepositoryCommand(Mock()) - user_input = { - cudl.OPT_UPSTREAM_NAME.keyword: 'pulp/crane', - } - result = command._parse_importer_config(user_input) - self.assertEqual(result[constants.CONFIG_KEY_UPSTREAM_NAME], 'pulp/crane') - - -class TestUpdateDockerRepositoryCommand(unittest.TestCase): - - def setUp(self): - self.context = Mock() - self.context.config = {'output': {'poll_frequency_in_seconds': 3}} - self.command = cudl.UpdateDockerRepositoryCommand(self.context) - self.command.poll = Mock() - self.mock_repo_response = Mock(response_body={}) - self.context.server.repo.repository.return_value = self.mock_repo_response - self.unit_search_command = Mock(response_body=[{u'metadata': {u'image_id': 'bar123'}}]) - self.context.server.repo_unit.search.return_value = self.unit_search_command - - def test_tag(self): - user_input = { - 'repo-id': 'foo-repo', - 'tag': [['foo', 'bar123']] - } - self.command.run(**user_input) - - repo_delta = { - u'scratchpad': {u'tags': [{constants.IMAGE_TAG_KEY: 'foo', - constants.IMAGE_ID_KEY: 'bar123'}]} - } - importer_config = None - dist_config = None - - self.context.server.repo.update.assert_called_once_with('foo-repo', repo_delta, - importer_config, dist_config) - - def test_run_with_importer_config(self): - user_input = { - 'repo-id': 'foo-repo', - cudl.OPT_UPSTREAM_NAME.keyword: 'pulp/crane', - } - self.command.run(**user_input) - - expected_importer_config = {constants.CONFIG_KEY_UPSTREAM_NAME: 'pulp/crane'} - - self.context.server.repo.update.assert_called_once_with('foo-repo', {}, - expected_importer_config, None) - - def test_tag_partial_match_image_id_too_short(self): - user_input = { - 'repo-id': 'foo-repo', - 'tag': [['foo', 'baz']] - } - self.unit_search_command.response_body = [{u'metadata': {u'image_id': 'baz123qux'}}] - self.command.run(**user_input) - self.assertFalse(self.context.server.repo.update.called) - - def test_tag_partial_match_image_id(self): - user_input = { - 'repo-id': 'foo-repo', - 'tag': [['foo', 'baz123']] - } - self.unit_search_command.response_body = [{u'metadata': {u'image_id': 'baz123qux'}}] - self.command.run(**user_input) - - target_kwargs = { - u'scratchpad': {u'tags': [{constants.IMAGE_TAG_KEY: 'foo', - constants.IMAGE_ID_KEY: 'baz123qux'}]} - } - self.context.server.repo.update.assert_called_once_with('foo-repo', target_kwargs, - None, None) - - def test_multi_tag(self): - user_input = { - 'repo-id': 'foo-repo', - 'tag': [['foo', 'bar123'], ['baz', 'bar123']] - } - self.command.run(**user_input) - - target_kwargs = { - u'scratchpad': {u'tags': [{constants.IMAGE_TAG_KEY: 'foo', - constants.IMAGE_ID_KEY: 'bar123'}, - {constants.IMAGE_TAG_KEY: 'baz', - constants.IMAGE_ID_KEY: 'bar123'}]} - } - self.context.server.repo.update.assert_called_once_with('foo-repo', target_kwargs, - None, None) - - def test_image_not_found(self): - user_input = { - 'repo-id': 'foo-repo', - 'tag': [['foo', 'bar123']] - } - self.unit_search_command.response_body = [] - self.command.run(**user_input) - self.assertTrue(self.command.prompt.render_failure_message.called) - - def test_remove_tag(self): - self.mock_repo_response.response_body = \ - {u'scratchpad': {u'tags': [{constants.IMAGE_TAG_KEY: 'foo', - constants.IMAGE_ID_KEY: 'bar123'}, - {constants.IMAGE_TAG_KEY: 'baz', - constants.IMAGE_ID_KEY: 'bar123'}]}} - user_input = { - 'repo-id': 'foo-repo', - 'remove-tag': ['foo'] - } - self.command.run(**user_input) - - target_kwargs = { - u'scratchpad': {u'tags': [{constants.IMAGE_TAG_KEY: 'baz', - constants.IMAGE_ID_KEY: 'bar123'}]} - } - self.context.server.repo.update.assert_called_once_with('foo-repo', target_kwargs, - None, None) - - def test_repo_update_distributors(self): - user_input = { - 'auto-publish': False, - 'repo-id': 'foo-repo', - 'protected': True, - 'repo-registry-id': 'flux', - 'redirect-url': 'bar' - } - self.command.run(**user_input) - - repo_config = {} - dist_config = {'docker_web_distributor_name_cli': {'protected': True, - 'auto_publish': False, - 'redirect-url': 'bar', - 'repo-registry-id': 'flux'}, - 'docker_export_distributor_name_cli': {'redirect-url': 'bar', - 'repo-registry-id': 'flux'}, - } - self.context.server.repo.update.assert_called_once_with('foo-repo', repo_config, - None, dist_config) diff --git a/extensions_admin/test/unit/extensions/admin/test_images.py b/extensions_admin/test/unit/extensions/admin/test_images.py deleted file mode 100644 index 0768e6a8..00000000 --- a/extensions_admin/test/unit/extensions/admin/test_images.py +++ /dev/null @@ -1,63 +0,0 @@ -import copy -import unittest - -from mock import MagicMock, patch - -from pulp_docker.common import constants -from pulp_docker.extensions.admin import images - - -class TestDockerImageCopyCommand(unittest.TestCase): - - @patch('pulp_docker.extensions.admin.images.get_formatter_for_type') - def test_get_formatter_for_type(self, mock_formatter): - context = MagicMock() - command = images.ImageCopyCommand(context) - - command.get_formatter_for_type('foo') - mock_formatter.assert_called_once_with('foo') - - -class TestGetFormatterForType(unittest.TestCase): - - def test_get_formatter_for_type(self): - formatter = images.get_formatter_for_type(constants.IMAGE_TYPE_ID) - self.assertEquals('foo', formatter({'image_id': 'foo'})) - - def test_get_formatter_for_type_raises_value_error(self): - self.assertRaises(ValueError, images.get_formatter_for_type, 'foo-type') - - -class TestImageRemoveCommand(unittest.TestCase): - - @patch('pulp_docker.extensions.admin.images.get_formatter_for_type') - def test_get_formatter_for_type(self, mock_formatter): - context = MagicMock() - command = images.ImageRemoveCommand(context) - - command.get_formatter_for_type('foo') - mock_formatter.assert_called_once_with('foo') - - -class TestImageSearchCommand(unittest.TestCase): - - def test_run(self): - context = MagicMock() - command = images.ImageSearchCommand(context) - - repo_info = { - u'scratchpad': {u'tags': [{constants.IMAGE_TAG_KEY: 'latest', - constants.IMAGE_ID_KEY: 'bar'}, - {constants.IMAGE_TAG_KEY: 'foo', - constants.IMAGE_ID_KEY: 'bar'}]} - } - context.server.repo.repository.return_value.response_body = repo_info - - image_list = [{u'metadata': {u'image_id': 'bar'}}] - context.server.repo_unit.search.return_value.response_body = image_list - - command.run(**{'repo-id': 'baz'}) - target = copy.deepcopy(image_list) - target[0][u'metadata'][u'tags'] = ['latest', 'foo'] - - context.prompt.render_document_list.assert_called_once_with(target) diff --git a/extensions_admin/test/unit/extensions/admin/test_parsers.py b/extensions_admin/test/unit/extensions/admin/test_parsers.py deleted file mode 100644 index 202cee55..00000000 --- a/extensions_admin/test/unit/extensions/admin/test_parsers.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest - -from pulp_docker.extensions.admin import parsers - - -class TestParseColonSeparated(unittest.TestCase): - - def test_with_value(self): - result = parsers.parse_colon_separated(['foo:bar']) - self.assertEquals(result, [['foo', 'bar']]) - - def test_with_none(self): - result = parsers.parse_colon_separated(None) - self.assertEquals(result, []) - - def test_with_no_colon(self): - self.assertRaises(ValueError, parsers.parse_colon_separated, ['bar']) - - def test_with_no_value_before_colon(self): - self.assertRaises(ValueError, parsers.parse_colon_separated, [':bar']) - - def test_with_no_value_after_colon(self): - self.assertRaises(ValueError, parsers.parse_colon_separated, ['foo:']) - - def test_with_multiple_colon(self): - result = parsers.parse_colon_separated(['foo:bar:baz']) - self.assertEquals(result, [['foo:bar', 'baz']]) diff --git a/extensions_admin/test/unit/extensions/admin/test_pulp_cli.py b/extensions_admin/test/unit/extensions/admin/test_pulp_cli.py deleted file mode 100644 index f351ff39..00000000 --- a/extensions_admin/test/unit/extensions/admin/test_pulp_cli.py +++ /dev/null @@ -1,52 +0,0 @@ -import unittest - -import mock -from pulp.client.commands.repo.cudl import CreateRepositoryCommand, DeleteRepositoryCommand -from pulp.client.commands.repo.cudl import UpdateRepositoryCommand -from pulp.client.commands.repo.sync_publish import PublishStatusCommand,\ - RunPublishRepositoryCommand, RunSyncRepositoryCommand -from pulp.client.commands.repo.upload import UploadCommand -from pulp.client.extensions.core import PulpCli - -from pulp_docker.extensions.admin import pulp_cli -from pulp_docker.extensions.admin import images -from pulp_docker.extensions.admin.repo_list import ListDockerRepositoriesCommand - - -class TestInitialize(unittest.TestCase): - def test_structure(self): - context = mock.MagicMock() - context.config = { - 'filesystem': {'upload_working_dir': '/a/b/c'}, - 'output': {'poll_frequency_in_seconds': 3} - } - context.cli = PulpCli(context) - - # create the tree of commands and sections - pulp_cli.initialize(context) - - # verify that sections exist and have the right commands - docker_section = context.cli.root_section.subsections['docker'] - - repo_section = docker_section.subsections['repo'] - self.assertTrue(isinstance(repo_section.commands['create'], CreateRepositoryCommand)) - self.assertTrue(isinstance(repo_section.commands['delete'], DeleteRepositoryCommand)) - self.assertTrue(isinstance(repo_section.commands['update'], UpdateRepositoryCommand)) - self.assertTrue(isinstance(repo_section.commands['list'], ListDockerRepositoriesCommand)) - self.assertTrue(isinstance(repo_section.commands['images'], images.ImageSearchCommand)) - self.assertTrue(isinstance(repo_section.commands['copy'], images.ImageCopyCommand)) - self.assertTrue(isinstance(repo_section.commands['remove'], images.ImageRemoveCommand)) - - upload_section = repo_section.subsections['uploads'] - self.assertTrue(isinstance(upload_section.commands['upload'], UploadCommand)) - - section = repo_section.subsections['sync'] - self.assertTrue(isinstance(section.commands['run'], RunSyncRepositoryCommand)) - - section = repo_section.subsections['publish'] - self.assertTrue(isinstance(section.commands['status'], PublishStatusCommand)) - self.assertTrue(isinstance(section.commands['run'], RunPublishRepositoryCommand)) - - section = repo_section.subsections['export'] - self.assertTrue(isinstance(section.commands['status'], PublishStatusCommand)) - self.assertTrue(isinstance(section.commands['run'], RunPublishRepositoryCommand)) diff --git a/extensions_admin/test/unit/extensions/admin/test_repo_list.py b/extensions_admin/test/unit/extensions/admin/test_repo_list.py deleted file mode 100644 index 9da5cd87..00000000 --- a/extensions_admin/test/unit/extensions/admin/test_repo_list.py +++ /dev/null @@ -1,105 +0,0 @@ -import unittest - -from mock import Mock -from pulp.common import constants as pulp_constants - -from pulp_docker.common import constants -from pulp_docker.extensions.admin.repo_list import ListDockerRepositoriesCommand - - -class TestListDockerRepositoriesCommand(unittest.TestCase): - def setUp(self): - self.context = Mock() - self.context.config = {'output': {'poll_frequency_in_seconds': 3}} - - def test_get_all_repos(self): - self.context.server.repo.repositories.return_value.response_body = 'foo' - command = ListDockerRepositoriesCommand(self.context) - result = command._all_repos({'bar': 'baz'}) - self.context.server.repo.repositories.assert_called_once_with({'bar': 'baz'}) - self.assertEquals('foo', result) - - def test_get_all_repos_caches_results(self): - command = ListDockerRepositoriesCommand(self.context) - command.all_repos_cache = 'foo' - result = command._all_repos({'bar': 'baz'}) - self.assertFalse(self.context.server.repo.repositories.called) - self.assertEquals('foo', result) - - def test_get_repositories(self): - # Setup - repos = [ - { - 'id': 'matching', - 'notes': {pulp_constants.REPO_NOTE_TYPE_KEY: constants.REPO_NOTE_DOCKER, }, - 'importers': [ - {'config': {}} - ], - 'distributors': [ - {'id': constants.CLI_EXPORT_DISTRIBUTOR_ID}, - {'id': constants.CLI_WEB_DISTRIBUTOR_ID} - ] - }, - {'id': 'non-rpm-repo', - 'notes': {}} - ] - self.context.server.repo.repositories.return_value.response_body = repos - - # Test - command = ListDockerRepositoriesCommand(self.context) - repos = command.get_repositories({}) - - # Verify - self.assertEqual(1, len(repos)) - self.assertEqual(repos[0]['id'], 'matching') - - # Make sure the export distributor was removed - self.assertEqual(len(repos[0]['distributors']), 1) - self.assertEqual(repos[0]['distributors'][0]['id'], constants.CLI_EXPORT_DISTRIBUTOR_ID) - - def test_get_repositories_no_details(self): - # Setup - repos = [ - { - 'id': 'foo', - 'display_name': 'bar', - 'notes': {pulp_constants.REPO_NOTE_TYPE_KEY: constants.REPO_NOTE_DOCKER, } - } - ] - self.context.server.repo.repositories.return_value.response_body = repos - - # Test - command = ListDockerRepositoriesCommand(self.context) - repos = command.get_repositories({}) - - # Verify - self.assertEqual(1, len(repos)) - self.assertEqual(repos[0]['id'], 'foo') - self.assertTrue('importers' not in repos[0]) - self.assertTrue('distributors' not in repos[0]) - - def test_get_other_repositories(self): - # Setup - repos = [ - { - 'repo_id': 'matching', - 'notes': {pulp_constants.REPO_NOTE_TYPE_KEY: constants.REPO_NOTE_DOCKER, }, - 'distributors': [ - {'id': constants.CLI_EXPORT_DISTRIBUTOR_ID}, - {'id': constants.CLI_WEB_DISTRIBUTOR_ID} - ] - }, - { - 'repo_id': 'non-rpm-repo-1', - 'notes': {} - } - ] - self.context.server.repo.repositories.return_value.response_body = repos - - # Test - command = ListDockerRepositoriesCommand(self.context) - repos = command.get_other_repositories({}) - - # Verify - self.assertEqual(1, len(repos)) - self.assertEqual(repos[0]['repo_id'], 'non-rpm-repo-1') diff --git a/extensions_admin/test/unit/extensions/admin/test_upload.py b/extensions_admin/test/unit/extensions/admin/test_upload.py deleted file mode 100644 index 6ede5f10..00000000 --- a/extensions_admin/test/unit/extensions/admin/test_upload.py +++ /dev/null @@ -1,44 +0,0 @@ -import unittest - -import mock - -from pulp_docker.common import constants -from pulp_docker.extensions.admin.upload import UploadDockerImageCommand, OPT_MASK_ANCESTOR_ID -import data - - -test_config = { - 'filesystem': {'upload_working_dir': '~/.pulp/upload/'}, - 'output': {'poll_frequency_in_seconds': 2}, -} - - -class TestUploadDockerImageCommand(unittest.TestCase): - def setUp(self): - self.context = mock.MagicMock() - self.context.config = test_config - self.command = UploadDockerImageCommand(self.context) - - def test_determine_id(self): - ret = self.command.determine_type_id('/a/b/c') - self.assertEqual(ret, constants.IMAGE_TYPE_ID) - - def test_generate_unit_key_and_metadata(self): - unit_key, metadata = self.command.generate_unit_key_and_metadata(data.busybox_tar_path) - self.assertEqual(unit_key, {}) - self.assertEqual(metadata, {}) - - def test_generate_override_config(self): - ret = self.command.generate_override_config() - self.assertEqual(ret, {}) - - def test_generate_override_config_with_mask_id(self): - test_mask_id = 'test-mask-id' - kwargs = {OPT_MASK_ANCESTOR_ID.keyword: test_mask_id} - ret = self.command.generate_override_config(**kwargs) - self.assertEqual(ret, {constants.CONFIG_KEY_MASK_ID: test_mask_id}) - - def test_generate_override_config_with_random_option(self): - kwargs = {'random': 'test_random_option'} - ret = self.command.generate_override_config(**kwargs) - self.assertEqual(ret, {}) diff --git a/flake8.cfg b/flake8.cfg index 88c235cf..57df5036 100644 --- a/flake8.cfg +++ b/flake8.cfg @@ -1,5 +1,19 @@ [flake8] -exclude = ./docs/* -# E401: multiple imports on one line -ignore = E401 +exclude = ./docs/*,*/migrations/* +ignore = Q000,D100,D104,D106,D200, D205, D400, D401,D402 max-line-length = 100 + +# Flake8-quotes extension codes +# ----------------------------- +# Q000: double or single quotes only, default is double (don't want to enforce this) + +# Flake8-docstring extension codes +# -------------------------------- +# D100: missing docstring in public module +# D104: missing docstring in public package +# D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) +# D200: one-line docstring should fit on one line with quotes +# D205: 1 blank line required between summary line and description +# D400: First line should end with a period +# D401: first line should be imperative (nitpicky) +# D402: first line should not be the function’s “signature” (false positives) diff --git a/functest_requirements.txt b/functest_requirements.txt new file mode 100644 index 00000000..f8ddc8f8 --- /dev/null +++ b/functest_requirements.txt @@ -0,0 +1,4 @@ +ecdsa~=0.13.2 +git+https://github.com/PulpQE/pulp-smash.git#egg=pulp-smash +pyjwkest~=1.4.0 +pulpcore~=3.0rc7 diff --git a/manage_setup_pys.sh b/manage_setup_pys.sh deleted file mode 100755 index dc4cf350..00000000 --- a/manage_setup_pys.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# -*- coding: utf-8 -*- - -# Use this bash script to run argument 1 on each setup.py in this project - -for setup in `find . -name setup.py`; do - pushd `dirname $setup`; - python setup.py "$@"; - popd; -done; diff --git a/plugins/etc/httpd/conf.d/pulp_docker.conf b/plugins/etc/httpd/conf.d/pulp_docker.conf deleted file mode 100644 index ffd8eff0..00000000 --- a/plugins/etc/httpd/conf.d/pulp_docker.conf +++ /dev/null @@ -1,15 +0,0 @@ -# -# Apache configuration file for Pulp's Docker support -# - -# -- HTTPS Repositories --------- - -Alias /pulp/docker /var/www/pub/docker/web - - - SSLRequireSSL - SSLVerifyClient optional_no_ca - SSLVerifyDepth 2 - SSLOptions +StdEnvVars +ExportCertData +FakeBasicAuth - Options FollowSymLinks Indexes - diff --git a/plugins/etc/pulp/server/plugins.conf.d/docker_distributor.json b/plugins/etc/pulp/server/plugins.conf.d/docker_distributor.json deleted file mode 100644 index 0db3279e..00000000 --- a/plugins/etc/pulp/server/plugins.conf.d/docker_distributor.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} diff --git a/plugins/etc/pulp/server/plugins.conf.d/docker_distributor_export.json b/plugins/etc/pulp/server/plugins.conf.d/docker_distributor_export.json deleted file mode 100644 index 0db3279e..00000000 --- a/plugins/etc/pulp/server/plugins.conf.d/docker_distributor_export.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} diff --git a/plugins/pulp_docker/__init__.py b/plugins/pulp_docker/__init__.py deleted file mode 100644 index 3ad9513f..00000000 --- a/plugins/pulp_docker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/plugins/pulp_docker/plugins/__init__.py b/plugins/pulp_docker/plugins/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/pulp_docker/plugins/distributors/__init__.py b/plugins/pulp_docker/plugins/distributors/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/pulp_docker/plugins/distributors/configuration.py b/plugins/pulp_docker/plugins/distributors/configuration.py deleted file mode 100644 index 444babc2..00000000 --- a/plugins/pulp_docker/plugins/distributors/configuration.py +++ /dev/null @@ -1,219 +0,0 @@ -import logging -import os -from urlparse import urlparse - -from pulp.server.config import config as server_config -from pulp.server.exceptions import PulpCodedValidationException - -from pulp_docker.common import constants, error_codes - -_LOG = logging.getLogger(__name__) - - -def validate_config(config): - """ - Validate a configuration - - :param config: Pulp configuration for the distributor - :type config: pulp.plugins.config.PluginCallConfiguration - :raises: PulpCodedValidationException if any validations failed - """ - errors = [] - server_url = config.get(constants.CONFIG_KEY_REDIRECT_URL) - if server_url: - parsed = urlparse(server_url) - if not parsed.scheme: - errors.append(PulpCodedValidationException(error_code=error_codes.DKR1001, - field=constants.CONFIG_KEY_REDIRECT_URL, - url=server_url)) - if not parsed.netloc: - errors.append(PulpCodedValidationException(error_code=error_codes.DKR1002, - field=constants.CONFIG_KEY_REDIRECT_URL, - url=server_url)) - if not parsed.path: - errors.append(PulpCodedValidationException(error_code=error_codes.DKR1003, - field=constants.CONFIG_KEY_REDIRECT_URL, - url=server_url)) - protected = config.get(constants.CONFIG_KEY_PROTECTED) - if protected: - protected_parsed = config.get_boolean(constants.CONFIG_KEY_PROTECTED) - if protected_parsed is None: - errors.append(PulpCodedValidationException(error_code=error_codes.DKR1004, - field=constants.CONFIG_KEY_PROTECTED, - value=protected)) - - if errors: - raise PulpCodedValidationException(validation_exceptions=errors) - - return True, None - - -def get_root_publish_directory(config): - """ - The publish directory for the docker plugin - - :param config: Pulp configuration for the distributor - :type config: pulp.plugins.config.PluginCallConfiguration - :return: The publish directory for the docker plugin - :rtype: str - """ - return config.get(constants.CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY) - - -def get_master_publish_dir(repo, config): - """ - Get the master publishing directory for the given repository. - This is the directory that links/files are actually published to - and linked from the directory published by the web server in an atomic action. - - :param repo: repository to get the master publishing directory for - :type repo: pulp.plugins.model.Repository - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or None - :return: master publishing directory for the given repository - :rtype: str - """ - return os.path.join(get_root_publish_directory(config), 'master', repo.id) - - -def get_web_publish_dir(repo, config): - """ - Get the configured HTTP publication directory. - Returns the global default if not configured. - - :param repo: repository to get relative path for - :type repo: pulp.plugins.model.Repository - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or None - - :return: the HTTP publication directory - :rtype: str - """ - - return os.path.join(get_root_publish_directory(config), - 'web', - get_repo_relative_path(repo, config)) - - -def get_app_publish_dir(config): - """ - Get the configured directory where the application redirect files should be stored - - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or None - - :returns: the name to use for the redirect file - :rtype: str - """ - return os.path.join(get_root_publish_directory(config), 'app',) - - -def get_redirect_file_name(repo): - """ - Get the name to use when generating the redirect file for a repository - - :param repo: the repository to get the app file name for - :type repo: pulp.plugins.model.Repository - - :returns: the name to use for the redirect file - :rtype: str - """ - return '%s.json' % repo.id - - -def get_redirect_url(config, repo): - """ - Get the redirect URL for a given repo & configuration - - :param config: configuration instance for the repository - :type config: pulp.plugins.config.PluginCallConfiguration or dict - :param repo: repository to get url for - :type repo: pulp.plugins.model.Repository - - """ - redirect_url = config.get(constants.CONFIG_KEY_REDIRECT_URL) - if redirect_url: - if not redirect_url.endswith('/'): - redirect_url += '/' - else: - # build the redirect URL from the server config - server_name = server_config.get('server', 'server_name') - redirect_url = 'https://%s/pulp/docker/%s/' % (server_name, repo.id) - - return redirect_url - - -def get_repo_relative_path(repo, config): - """ - Get the configured relative path for the given repository. - - :param repo: repository to get relative path for - :type repo: pulp.plugins.model.Repository - :param config: configuration instance for the repository - :type config: pulp.plugins.config.PluginCallConfiguration or dict - :return: relative path for the repository - :rtype: str - """ - return repo.id - - -def get_export_repo_directory(config): - """ - Get the directory where the export publisher will publish repositories. - - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or NoneType - :return: directory where export files are saved - :rtype: str - """ - return os.path.join(get_root_publish_directory(config), 'export', 'repo') - - -def get_export_repo_filename(repo, config): - """ - Get the file name for a repository export - - :param repo: repository being exported - :type repo: pulp.plugins.model.Repository - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or NoneType - :return: The file name for the published tar file - :rtype: str - """ - return '%s.tar' % repo.id - - -def get_export_repo_file_with_path(repo, config): - """ - Get the file name to use when exporting a docker repo as a tar file - - :param repo: repository being exported - :type repo: pulp.plugins.model.Repository - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or NoneType - :return: The absolute file name for the tar file that will be exported - :rtype: str - """ - file_name = config.get(constants.CONFIG_KEY_EXPORT_FILE) - if not file_name: - file_name = os.path.join(get_export_repo_directory(config), - get_export_repo_filename(repo, config)) - return file_name - - -def get_repo_registry_id(repo, config): - """ - Get the registry ID that should be used by the docker API. If a registry name has not - been specified on the repo fail back to the repo id. - - :param repo: repository to get relative path for - :type repo: pulp.plugins.model.Repository - :param config: configuration instance - :type config: pulp.plugins.config.PluginCallConfiguration or NoneType - :return: The name of the repository as it should be represented in in the Docker API - :rtype: str - """ - registry = config.get(constants.CONFIG_KEY_REPO_REGISTRY_ID) - if not registry: - registry = repo.id - return registry diff --git a/plugins/pulp_docker/plugins/distributors/distributor_export.py b/plugins/pulp_docker/plugins/distributors/distributor_export.py deleted file mode 100644 index 4978e99e..00000000 --- a/plugins/pulp_docker/plugins/distributors/distributor_export.py +++ /dev/null @@ -1,167 +0,0 @@ -from gettext import gettext as _ -import copy -import logging -import os -import shutil - -from pulp.common.config import read_json_config -from pulp.plugins.distributor import Distributor - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors.publish_steps import ExportPublisher -from pulp_docker.plugins.distributors import configuration - - -_logger = logging.getLogger(__name__) - -PLUGIN_DEFAULT_CONFIG = { - constants.CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY: constants.CONFIG_VALUE_DOCKER_PUBLISH_DIRECTORY -} - - -def entry_point(): - """ - Entry point that pulp platform uses to load the distributor - :return: distributor class and its config - :rtype: Distributor, dict - """ - plugin_config = copy.deepcopy(PLUGIN_DEFAULT_CONFIG) - edited_config = read_json_config(constants.DISTRIBUTOR_EXPORT_CONFIG_FILE_NAME) - - plugin_config.update(edited_config) - return DockerExportDistributor, plugin_config - - -class DockerExportDistributor(Distributor): - @classmethod - def metadata(cls): - """ - Used by Pulp to classify the capabilities of this distributor. The - following keys must be present in the returned dictionary: - - * id - Programmatic way to refer to this distributor. Must be unique - across all distributors. Only letters and underscores are valid. - * display_name - User-friendly identification of the distributor. - * types - List of all content type IDs that may be published using this - distributor. - - :return: keys and values listed above - :rtype: dict - """ - return { - 'id': constants.DISTRIBUTOR_EXPORT_TYPE_ID, - 'display_name': _('Docker Export Distributor'), - 'types': [constants.IMAGE_TYPE_ID] - } - - def __init__(self): - super(DockerExportDistributor, self).__init__() - self._publisher = None - self.canceled = False - - def validate_config(self, repo, config, config_conduit): - """ - Allows the distributor to check the contents of a potential configuration - for the given repository. This call is made both for the addition of - this distributor to a new repository as well as updating the configuration - for this distributor on a previously configured repository. The implementation - should use the given repository data to ensure that updating the - configuration does not put the repository into an inconsistent state. - - The return is a tuple of the result of the validation (True for success, - False for failure) and a message. The message may be None and is unused - in the success case. For a failed validation, the message will be - communicated to the caller so the plugin should take i18n into - consideration when generating the message. - - The related_repos parameter contains a list of other repositories that - have a configured distributor of this type. The distributor configurations - is found in each repository in the "plugin_configs" field. - - :param repo: metadata describing the repository to which the - configuration applies - :type repo: pulp.plugins.model.Repository - - :param config: plugin configuration instance; the proposed repo - configuration is found within - :type config: pulp.plugins.config.PluginCallConfiguration - - :param config_conduit: Configuration Conduit; - :type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit - - :return: tuple of (bool, str) to describe the result - :rtype: tuple - """ - return configuration.validate_config(config) - - def publish_repo(self, repo, publish_conduit, config): - """ - Publishes the given repository. - - While this call may be implemented using multiple threads, its execution - from the Pulp server's standpoint should be synchronous. This call should - not return until the publish is complete. - - It is not expected that this call be atomic. Should an error occur, it - is not the responsibility of the distributor to rollback any changes - that have been made. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - - :param publish_conduit: provides access to relevant Pulp functionality - :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginConfiguration - - :return: report describing the publish run - :rtype: pulp.plugins.model.PublishReport - """ - self._publisher = ExportPublisher(repo, publish_conduit, config) - return self._publisher.publish() - - def cancel_publish_repo(self): - """ - Call cancellation control hook. - """ - self.canceled = True - if self._publisher is not None: - self._publisher.cancel() - - def distributor_removed(self, repo, config): - """ - Called when a distributor of this type is removed from a repository. - This hook allows the distributor to clean up any files that may have - been created during the actual publishing. - - The distributor may use the contents of the working directory in cleanup. - It is not required that the contents of this directory be deleted by - the distributor; Pulp will ensure it is wiped following this call. - - If this call raises an exception, the distributor will still be removed - from the repository and the working directory contents will still be - wiped by Pulp. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - """ - # remove the directories that might have been created for this repo/distributor - dir_list = [repo.working_dir] - - for repo_dir in dir_list: - shutil.rmtree(repo_dir, ignore_errors=True) - - # Remove the published app file & directory links - file_list = [os.path.join(configuration.get_export_repo_directory(config), - configuration.get_export_repo_filename(repo, config))] - - for file_name in file_list: - try: - os.unlink(file_name) - except OSError: - # It's fine if this file doesn't exist - pass diff --git a/plugins/pulp_docker/plugins/distributors/distributor_web.py b/plugins/pulp_docker/plugins/distributors/distributor_web.py deleted file mode 100644 index e808e863..00000000 --- a/plugins/pulp_docker/plugins/distributors/distributor_web.py +++ /dev/null @@ -1,172 +0,0 @@ -from gettext import gettext as _ -import copy -import logging -import os -import shutil - -from pulp.common.config import read_json_config -from pulp.plugins.distributor import Distributor - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors.publish_steps import WebPublisher -from pulp_docker.plugins.distributors import configuration - - -_logger = logging.getLogger(__name__) - -PLUGIN_DEFAULT_CONFIG = { - constants.CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY: constants.CONFIG_VALUE_DOCKER_PUBLISH_DIRECTORY -} - - -def entry_point(): - """ - Entry point that pulp platform uses to load the distributor - :return: distributor class and its config - :rtype: Distributor, dict - """ - plugin_config = copy.deepcopy(PLUGIN_DEFAULT_CONFIG) - edited_config = read_json_config(constants.DISTRIBUTOR_CONFIG_FILE_NAME) - - plugin_config.update(edited_config) - return DockerWebDistributor, plugin_config - - -class DockerWebDistributor(Distributor): - @classmethod - def metadata(cls): - """ - Used by Pulp to classify the capabilities of this distributor. The - following keys must be present in the returned dictionary: - - * id - Programmatic way to refer to this distributor. Must be unique - across all distributors. Only letters and underscores are valid. - * display_name - User-friendly identification of the distributor. - * types - List of all content type IDs that may be published using this - distributor. - - :return: keys and values listed above - :rtype: dict - """ - return { - 'id': constants.DISTRIBUTOR_WEB_TYPE_ID, - 'display_name': _('Docker Web Distributor'), - 'types': [constants.IMAGE_TYPE_ID] - } - - def __init__(self): - super(DockerWebDistributor, self).__init__() - self._publisher = None - self.canceled = False - - def validate_config(self, repo, config, config_conduit): - """ - Allows the distributor to check the contents of a potential configuration - for the given repository. This call is made both for the addition of - this distributor to a new repository as well as updating the configuration - for this distributor on a previously configured repository. The implementation - should use the given repository data to ensure that updating the - configuration does not put the repository into an inconsistent state. - - The return is a tuple of the result of the validation (True for success, - False for failure) and a message. The message may be None and is unused - in the success case. For a failed validation, the message will be - communicated to the caller so the plugin should take i18n into - consideration when generating the message. - - The related_repos parameter contains a list of other repositories that - have a configured distributor of this type. The distributor configurations - is found in each repository in the "plugin_configs" field. - - :param repo: metadata describing the repository to which the - configuration applies - :type repo: pulp.plugins.model.Repository - - :param config: plugin configuration instance; the proposed repo - configuration is found within - :type config: pulp.plugins.config.PluginCallConfiguration - - :param config_conduit: Configuration Conduit; - :type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit - - :return: tuple of (bool, str) to describe the result - :rtype: tuple - """ - return configuration.validate_config(config) - - def publish_repo(self, repo, publish_conduit, config): - """ - Publishes the given repository. - - While this call may be implemented using multiple threads, its execution - from the Pulp server's standpoint should be synchronous. This call should - not return until the publish is complete. - - It is not expected that this call be atomic. Should an error occur, it - is not the responsibility of the distributor to rollback any changes - that have been made. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - - :param publish_conduit: provides access to relevant Pulp functionality - :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginConfiguration - - :return: report describing the publish run - :rtype: pulp.plugins.model.PublishReport - """ - _logger.debug('Publishing docker repository: %s' % repo.id) - self._publisher = WebPublisher(repo, publish_conduit, config) - return self._publisher.publish() - - def cancel_publish_repo(self): - """ - Call cancellation control hook. - """ - _logger.debug('Canceling docker repository publish') - self.canceled = True - if self._publisher is not None: - self._publisher.cancel() - - def distributor_removed(self, repo, config): - """ - Called when a distributor of this type is removed from a repository. - This hook allows the distributor to clean up any files that may have - been created during the actual publishing. - - The distributor may use the contents of the working directory in cleanup. - It is not required that the contents of this directory be deleted by - the distributor; Pulp will ensure it is wiped following this call. - - If this call raises an exception, the distributor will still be removed - from the repository and the working directory contents will still be - wiped by Pulp. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - """ - # remove the directories that might have been created for this repo/distributor - dir_list = [repo.working_dir, - configuration.get_master_publish_dir(repo, config), - configuration.get_web_publish_dir(repo, config)] - - for repo_dir in dir_list: - shutil.rmtree(repo_dir, ignore_errors=True) - - # Remove the published app file & directory links - dir_list = [configuration.get_web_publish_dir(repo, config), - os.path.join(configuration.get_app_publish_dir(config), - configuration.get_redirect_file_name(repo))] - - for repo_dir in dir_list: - try: - os.unlink(repo_dir) - except OSError: - # It's fine if this file doesn't exist - pass diff --git a/plugins/pulp_docker/plugins/distributors/metadata.py b/plugins/pulp_docker/plugins/distributors/metadata.py deleted file mode 100644 index cae22c7c..00000000 --- a/plugins/pulp_docker/plugins/distributors/metadata.py +++ /dev/null @@ -1,97 +0,0 @@ -import logging -import os - - -from pulp.server.compat import json -from pulp.plugins.util.metadata_writer import JSONArrayFileContext - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors import configuration - -_LOG = logging.getLogger(__name__) - - -class RedirectFileContext(JSONArrayFileContext): - """ - Context manager for generating the docker images file. - """ - - def __init__(self, working_dir, conduit, config, repo): - """ - :param working_dir: working directory to create the filelists.xml.gz in - :type working_dir: str - :param conduit: The conduit to get api calls - :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - :param config: Pulp configuration for the distributor - :type config: pulp.plugins.config.PluginCallConfiguration - :param repo: Pulp managed repository - :type repo: pulp.plugins.model.Repository - """ - - self.repo_id = repo.id - metadata_file_path = os.path.join(working_dir, - configuration.get_redirect_file_name(repo)) - super(RedirectFileContext, self).__init__(metadata_file_path) - scratchpad = conduit.get_repo_scratchpad() - - tag_list = scratchpad.get(u'tags', []) - self.tags = self.convert_tag_list_to_dict(tag_list) - - self.registry = configuration.get_repo_registry_id(repo, config) - - self.redirect_url = configuration.get_redirect_url(config, repo) - if config.get('protected', False): - self.protected = "true" - else: - self.protected = "false" - - def _write_file_header(self): - """ - Write out the beginning of the json file - """ - self.metadata_file_handle.write('{"type":"pulp-docker-redirect","version":1,' - '"repository":"%s","repo-registry-id": "%s",' - '"url":"%s","protected":%s,"images":[' % - (self.repo_id, self.registry, self.redirect_url, - self.protected)) - - def _write_file_footer(self): - """ - Write out the end of the json file - """ - self.metadata_file_handle.write('],"tags":') - self.metadata_file_handle.write(json.dumps(self.tags)) - self.metadata_file_handle.write('}') - - def add_unit_metadata(self, unit): - """ - Add the specific metadata for this unit - - :param unit: The docker unit to add to the images metadata file - :type unit: pulp.plugins.model.AssociatedUnit - """ - super(RedirectFileContext, self).add_unit_metadata(unit) - image_id = unit.unit_key['image_id'] - unit_data = { - 'id': image_id - } - string_representation = json.dumps(unit_data) - self.metadata_file_handle.write(string_representation) - - def convert_tag_list_to_dict(self, tag_list): - """ - Convert a list of tags to a dictionary with tag as the key and image id as value. - If a single tag is associated with multiple image_ids, they will be overwritten. - Since we make sure this doesn't happen when adding image tags to a repository, - we can safely do the conversion. - - :param tag_list: list of dictionaries each containing values for 'tag' and 'image_id' keys - :type tag_list: list of dict - - :return: dictionary of tag:image_id - :rtype: dict - """ - tag_dict = {} - for tag in tag_list: - tag_dict[tag[constants.IMAGE_TAG_KEY]] = tag[constants.IMAGE_ID_KEY] - return tag_dict diff --git a/plugins/pulp_docker/plugins/distributors/publish_steps.py b/plugins/pulp_docker/plugins/distributors/publish_steps.py deleted file mode 100644 index 0d2372e7..00000000 --- a/plugins/pulp_docker/plugins/distributors/publish_steps.py +++ /dev/null @@ -1,119 +0,0 @@ -from gettext import gettext as _ -import logging -import os - -from pulp.plugins.util.publish_step import PublishStep, UnitPublishStep, \ - AtomicDirectoryPublishStep, SaveTarFilePublishStep - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors import configuration -from pulp_docker.plugins.distributors.metadata import RedirectFileContext - - -_LOG = logging.getLogger(__name__) - - -class WebPublisher(PublishStep): - """ - Docker Web publisher class that is responsible for the actual publishing - of a docker repository via a web server - """ - - def __init__(self, repo, publish_conduit, config): - """ - :param repo: Pulp managed Yum repository - :type repo: pulp.plugins.model.Repository - :param publish_conduit: Conduit providing access to relative Pulp functionality - :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - :param config: Pulp configuration for the distributor - :type config: pulp.plugins.config.PluginCallConfiguration - """ - super(WebPublisher, self).__init__(constants.PUBLISH_STEP_WEB_PUBLISHER, - repo, publish_conduit, config) - - publish_dir = configuration.get_web_publish_dir(repo, config) - app_file = configuration.get_redirect_file_name(repo) - app_publish_location = os.path.join(configuration.get_app_publish_dir(config), app_file) - self.web_working_dir = os.path.join(self.get_working_dir(), 'web') - master_publish_dir = configuration.get_master_publish_dir(repo, config) - atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), - [('web', publish_dir), - (app_file, app_publish_location)], - master_publish_dir, - step_type=constants.PUBLISH_STEP_OVER_HTTP) - atomic_publish_step.description = _('Making files available via web.') - self.add_child(PublishImagesStep()) - self.add_child(atomic_publish_step) - - -class ExportPublisher(PublishStep): - """ - Docker Export publisher class that is responsible for the actual publishing - of a docker repository via a tar file - """ - - def __init__(self, repo, publish_conduit, config): - """ - :param repo: Pulp managed Yum repository - :type repo: pulp.plugins.model.Repository - :param publish_conduit: Conduit providing access to relative Pulp functionality - :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - :param config: Pulp configuration for the distributor - :type config: pulp.plugins.config.PluginCallConfiguration - """ - super(ExportPublisher, self).__init__(constants.PUBLISH_STEP_EXPORT_PUBLISHER, - repo, publish_conduit, config) - - self.add_child(PublishImagesStep()) - tar_file = configuration.get_export_repo_file_with_path(repo, config) - self.add_child(SaveTarFilePublishStep(self.get_working_dir(), tar_file)) - - -class PublishImagesStep(UnitPublishStep): - """ - Publish Images - """ - - def __init__(self): - super(PublishImagesStep, self).__init__(constants.PUBLISH_STEP_IMAGES, - constants.IMAGE_TYPE_ID) - self.context = None - self.redirect_context = None - self.description = _('Publishing Image Files.') - - def initialize(self): - """ - Initialize the metadata contexts - """ - self.redirect_context = RedirectFileContext(self.get_working_dir(), - self.get_conduit(), - self.parent.config, - self.get_repo()) - self.redirect_context.initialize() - - def process_unit(self, unit): - """ - Link the unit to the image content directory and the package_dir - - :param unit: The unit to process - :type unit: pulp_docker.common.models.DockerImage - """ - self.redirect_context.add_unit_metadata(unit) - target_base = os.path.join(self.get_web_directory(), unit.unit_key['image_id']) - files = ['ancestry', 'json', 'layer'] - for file_name in files: - self._create_symlink(os.path.join(unit.storage_path, file_name), - os.path.join(target_base, file_name)) - - def finalize(self): - """ - Close & finalize each the metadata context - """ - if self.redirect_context: - self.redirect_context.finalize() - - def get_web_directory(self): - """ - Get the directory where the files published to the web have been linked - """ - return os.path.join(self.get_working_dir(), 'web') diff --git a/plugins/pulp_docker/plugins/importers/__init__.py b/plugins/pulp_docker/plugins/importers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/pulp_docker/plugins/importers/importer.py b/plugins/pulp_docker/plugins/importers/importer.py deleted file mode 100644 index 362b7bbc..00000000 --- a/plugins/pulp_docker/plugins/importers/importer.py +++ /dev/null @@ -1,255 +0,0 @@ -from gettext import gettext as _ -import logging -import tempfile - -from pulp.common.config import read_json_config -from pulp.plugins.conduits.mixins import UnitAssociationCriteria -from pulp.plugins.importer import Importer -import pulp.server.managers.factory as manager_factory -import shutil - -from pulp_docker.common import constants, tarutils -from pulp_docker.plugins.importers import upload -from pulp_docker.plugins.importers import sync - - -_logger = logging.getLogger(__name__) - - -def entry_point(): - """ - Entry point that pulp platform uses to load the importer - :return: importer class and its config - :rtype: Importer, dict - """ - plugin_config = read_json_config(constants.IMPORTER_CONFIG_FILE_NAME) - return DockerImporter, plugin_config - - -class DockerImporter(Importer): - @classmethod - def metadata(cls): - """ - Used by Pulp to classify the capabilities of this importer. The - following keys must be present in the returned dictionary: - - * id - Programmatic way to refer to this importer. Must be unique - across all importers. Only letters and underscores are valid. - * display_name - User-friendly identification of the importer. - * types - List of all content type IDs that may be imported using this - importer. - - :return: keys and values listed above - :rtype: dict - """ - return { - 'id': constants.IMPORTER_TYPE_ID, - 'display_name': _('Docker Importer'), - 'types': [constants.IMAGE_TYPE_ID] - } - - def sync_repo(self, repo, sync_conduit, config): - """ - Synchronizes content into the given repository. This call is responsible - for adding new content units to Pulp as well as associating them to the - given repository. - - While this call may be implemented using multiple threads, its execution - from the Pulp server's standpoint should be synchronous. This call should - not return until the sync is complete. - - It is not expected that this call be atomic. Should an error occur, it - is not the responsibility of the importer to rollback any unit additions - or associations that have been made. - - The returned report object is used to communicate the results of the - sync back to the user. Care should be taken to i18n the free text "log" - attribute in the report if applicable. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - - :param sync_conduit: provides access to relevant Pulp functionality - :type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - - :return: report of the details of the sync - :rtype: pulp.plugins.model.SyncReport - """ - working_dir = tempfile.mkdtemp(dir=repo.working_dir) - try: - self.sync_step = sync.SyncStep(repo=repo, conduit=sync_conduit, config=config, - working_dir=working_dir) - return self.sync_step.sync() - - finally: - shutil.rmtree(working_dir, ignore_errors=True) - - def cancel_sync_repo(self): - """ - Cancels an in-progress sync. - - This call is responsible for halting a current sync by stopping any - in-progress downloads and performing any cleanup necessary to get the - system back into a stable state. - """ - self.sync_step.cancel() - - def upload_unit(self, repo, type_id, unit_key, metadata, file_path, conduit, config): - """ - Upload a docker image. The file should be the product of "docker save". - This will import all images in that tarfile into the specified - repository, each as an individual unit. This will also update the - repo's tags to reflect the tags present in the tarfile. - - The following is copied from the superclass. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - :param type_id: type of unit being uploaded - :type type_id: str - :param unit_key: identifier for the unit, specified by the user - :type unit_key: dict - :param metadata: any user-specified metadata for the unit - :type metadata: dict - :param file_path: path on the Pulp server's filesystem to the temporary location of the - uploaded file; may be None in the event that a unit is comprised entirely - of metadata and has no bits associated - :type file_path: str - :param conduit: provides access to relevant Pulp functionality - :type conduit: pulp.plugins.conduits.unit_add.UnitAddConduit - :param config: plugin configuration for the repository - :type config: pulp.plugins.config.PluginCallConfiguration - :return: A dictionary describing the success or failure of the upload. It must - contain the following keys: - 'success_flag': bool. Indicates whether the upload was successful - 'summary': json-serializable object, providing summary - 'details': json-serializable object, providing details - :rtype: dict - """ - # retrieve metadata from the tarball - metadata = tarutils.get_metadata(file_path) - # turn that metadata into a collection of models - mask_id = config.get(constants.CONFIG_KEY_MASK_ID) - models = upload.get_models(metadata, mask_id) - ancestry = tarutils.get_ancestry(models[0].image_id, metadata) - # save those models as units in pulp - upload.save_models(conduit, models, ancestry, file_path) - upload.update_tags(repo.id, file_path) - - def import_units(self, source_repo, dest_repo, import_conduit, config, units=None): - """ - Import content units into the given repository. This method will be - called in a number of different situations: - * A user is attempting to copy a content unit from one repository - into the repository that uses this importer - * A user is attempting to add an orphaned unit into a repository. - - This call has two options for handling the requested units: - * Associate the given units with the destination repository. This will - link the repository with the existing unit directly; changes to the - unit will be reflected in all repositories that reference it. - * Create a new unit and save it to the repository. This would act as - a deep copy of sorts, creating a unique unit in the database. Keep - in mind that the unit key must change in order for the unit to - be considered different than the supplied one. - - The APIs for both approaches are similar to those in the sync conduit. - In the case of a simple association, the init_unit step can be skipped - and save_unit simply called on each specified unit. - - The units argument is optional. If None, all units in the source - repository should be imported. The conduit is used to query for those - units. If specified, only the units indicated should be imported (this - is the case where the caller passed a filter to Pulp). - - :param source_repo: metadata describing the repository containing the - units to import - :type source_repo: pulp.plugins.model.Repository - - :param dest_repo: metadata describing the repository to import units - into - :type dest_repo: pulp.plugins.model.Repository - - :param import_conduit: provides access to relevant Pulp functionality - :type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - - :param units: optional list of pre-filtered units to import - :type units: list of pulp.plugins.model.Unit - - :return: list of Unit instances that were saved to the destination repository - :rtype: list - """ - # Determine which units are being copied - if units is None: - criteria = UnitAssociationCriteria(type_ids=[constants.IMAGE_TYPE_ID]) - units = import_conduit.get_source_units(criteria=criteria) - - # Associate to the new repository - known_units = set() - units_added = [] - - while True: - units_to_add = set() - - # Associate the units to the repository - for u in units: - import_conduit.associate_unit(u) - units_added.append(u) - known_units.add(u.unit_key['image_id']) - parent_id = u.metadata.get('parent_id') - if parent_id: - units_to_add.add(parent_id) - # Filter out units we have already added - units_to_add.difference_update(known_units) - # Find any new units to add to the repository - if units_to_add: - unit_filter = {'image_id': {'$in': list(units_to_add)}} - criteria = UnitAssociationCriteria(type_ids=[constants.IMAGE_TYPE_ID], - unit_filters=unit_filter) - units = import_conduit.get_source_units(criteria=criteria) - else: - # Break out of the loop since there were no units to add to the list - break - - return units_added - - def validate_config(self, repo, config): - """ - We don't have a config yet, so it's always valid - """ - return True, '' - - def remove_units(self, repo, units, config): - """ - Removes content units from the given repository. - - This method also removes the tags associated with images in the repository. - - This call will not result in the unit being deleted from Pulp itself. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - - :param units: list of objects describing the units to import in - this call - :type units: list of pulp.plugins.model.AssociatedUnit - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - """ - repo_manager = manager_factory.repo_manager() - scratchpad = repo_manager.get_repo_scratchpad(repo.id) - tags = scratchpad.get(u'tags', []) - unit_ids = set([unit.unit_key[u'image_id'] for unit in units]) - for tag_dict in tags[:]: - if tag_dict[constants.IMAGE_ID_KEY] in unit_ids: - tags.remove(tag_dict) - - scratchpad[u'tags'] = tags - repo_manager.set_repo_scratchpad(repo.id, scratchpad) diff --git a/plugins/pulp_docker/plugins/importers/sync.py b/plugins/pulp_docker/plugins/importers/sync.py deleted file mode 100644 index 69ac9872..00000000 --- a/plugins/pulp_docker/plugins/importers/sync.py +++ /dev/null @@ -1,278 +0,0 @@ -import errno -from gettext import gettext as _ -import json -import logging -import os -import shutil - -from pulp.common.plugins import importer_constants -from pulp.plugins.util import nectar_config -from pulp.plugins.util.publish_step import PluginStep, DownloadStep, \ - GetLocalUnitsStep - -from pulp_docker.common import constants -from pulp_docker.common.models import DockerImage -from pulp_docker.plugins.importers import tags -from pulp_docker.plugins.registry import Repository - - -_logger = logging.getLogger(__name__) - - -class SyncStep(PluginStep): - def __init__(self, repo=None, conduit=None, config=None, - working_dir=None): - """ - :param repo: repository to sync - :type repo: pulp.plugins.model.Repository - :param conduit: sync conduit to use - :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit - :param config: config object for the sync - :type config: pulp.plugins.config.PluginCallConfiguration - :param working_dir: full path to the directory in which transient files - should be stored before being moved into long-term - storage. This should be deleted by the caller after - step processing is complete. - :type working_dir: basestring - """ - super(SyncStep, self).__init__(constants.SYNC_STEP_MAIN, repo, conduit, config, - working_dir, constants.IMPORTER_TYPE_ID) - self.description = _('Syncing Docker Repository') - - # Unit keys, populated by GetMetadataStep - self.available_units = [] - # populated by GetMetadataStep - self.tags = {} - - # create a Repository object to interact with - download_config = nectar_config.importer_config_to_nectar_config(config.flatten()) - upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME) - url = config.get(importer_constants.KEY_FEED) - self.index_repository = Repository(upstream_name, download_config, url, working_dir) - - self.add_child(GetMetadataStep(working_dir=working_dir)) - # save this step so its "units_to_download" attribute can be accessed later - self.step_get_local_units = GetLocalImagesStep(constants.IMPORTER_TYPE_ID, - constants.IMAGE_TYPE_ID, - ['image_id'], working_dir) - self.add_child(self.step_get_local_units) - self.add_child(DownloadStep(constants.SYNC_STEP_DOWNLOAD, - downloads=self.generate_download_requests(), - repo=repo, config=config, working_dir=working_dir, - description=_('Downloading remote files'))) - self.add_child(SaveUnits(working_dir)) - - def generate_download_requests(self): - """ - a generator that yields DownloadRequest objects based on which units - were determined to be needed. This looks at the GetLocalUnits step's - output, which includes a list of units that need their files downloaded. - - :return: generator of DownloadRequest instances - :rtype: types.GeneratorType - """ - for unit_key in self.step_get_local_units.units_to_download: - image_id = unit_key['image_id'] - destination_dir = os.path.join(self.get_working_dir(), image_id) - try: - os.makedirs(destination_dir, mode=0755) - except OSError, e: - # it's ok if the directory exists - if e.errno != errno.EEXIST: - raise - # we already retrieved the ancestry files for the tagged images, so - # some of these will already exist - if not os.path.exists(os.path.join(destination_dir, 'ancestry')): - yield self.index_repository.create_download_request(image_id, 'ancestry', - destination_dir) - - yield self.index_repository.create_download_request(image_id, 'json', destination_dir) - yield self.index_repository.create_download_request(image_id, 'layer', destination_dir) - - def sync(self): - """ - actually initiate the sync - - :return: a final sync report - :rtype: pulp.plugins.model.SyncReport - """ - self.process_lifecycle() - return self._build_final_report() - - -class GetMetadataStep(PluginStep): - def __init__(self, repo=None, conduit=None, config=None, working_dir=None): - """ - :param repo: repository to sync - :type repo: pulp.plugins.model.Repository - :param conduit: sync conduit to use - :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit - :param config: config object for the sync - :type config: pulp.plugins.config.PluginCallConfiguration - :param working_dir: full path to the directory in which transient files - should be stored before being moved into long-term - storage. This should be deleted by the caller after - step processing is complete. - :type working_dir: basestring - """ - super(GetMetadataStep, self).__init__(constants.SYNC_STEP_METADATA, repo, conduit, config, - working_dir, constants.IMPORTER_TYPE_ID) - self.description = _('Retrieving metadata') - - def process_main(self): - """ - determine what images are available upstream, get the upstream tags, and - save a list of available unit keys on the parent step - """ - super(GetMetadataStep, self).process_main() - download_dir = self.get_working_dir() - _logger.debug(self.description) - - # determine what images are available by querying the upstream source - available_images = self.parent.index_repository.get_image_ids() - # get remote tags and save them on the parent - self.parent.tags.update(self.parent.index_repository.get_tags()) - # transform the tags so they contain full image IDs instead of abbreviations - self.expand_tag_abbreviations(available_images, self.parent.tags) - - tagged_image_ids = self.parent.tags.values() - - # retrieve ancestry files and then parse them to determine the full - # collection of upstream images that we should ensure are obtained. - self.parent.index_repository.get_ancestry(tagged_image_ids) - images_we_need = set(tagged_image_ids) - for image_id in tagged_image_ids: - images_we_need.update(set(self.find_and_read_ancestry_file(image_id, download_dir))) - - # generate unit keys and save them on the parent - self.parent.available_units = [dict(image_id=i) for i in images_we_need] - - @staticmethod - def expand_tag_abbreviations(image_ids, tags): - """ - Given a list of full image IDs and a dictionary of tags, where the values - are either image IDs or abbreviated image IDs, this function replaces - abbreviated image IDs in the tags dictionary with full IDs. Changes are - applied in-place to the passed-in dictionary. - - This algorithm will not scale well, but it's unlikely we'll ever see - n>100, let alone a scale where this algorithm would become a bottleneck. - For such small data sets, a fancier and more efficient algorithm would - require enough setup and custom data structures, that the overhead might - often outweigh any gains. - - :param image_ids: list of image IDs - :type image_ids: list - :param tags: dictionary where keys are tag names and values are - either full image IDs or abbreviated image IDs. - """ - for tag_name, abbreviated_id in tags.items(): - for image_id in image_ids: - if image_id.startswith(abbreviated_id): - tags[tag_name] = image_id - break - - @staticmethod - def find_and_read_ancestry_file(image_id, parent_dir): - """ - Given an image ID, find it's file directory in the given parent directory - (it will be a directory whose name in the image_id), open the "ancestry" - file within it, deserialize its contents as json, and return the result. - - :param image_id: unique ID of a docker image - :type image_id: basestring - :param parent_dir: full path to the parent directory in which we should - look for a directory whose name is the image_id - :type parent_dir: basestring - - :return: list of image_ids that represent the ancestry for the image ID - :rtype: list - """ - with open(os.path.join(parent_dir, image_id, 'ancestry')) as ancestry_file: - return json.load(ancestry_file) - - -class GetLocalImagesStep(GetLocalUnitsStep): - def _dict_to_unit(self, unit_dict): - """ - convert a unit dictionary (a flat dict that has all unit key, metadata, - etc. keys at the root level) into a Unit object. This requires knowing - not just what fields are part of the unit key, but also how to derive - the storage path. - - :param unit_dict: a flat dictionary that has all unit key, metadata, - etc. keys at the root level, representing a unit - in pulp - :type unit_dict: dict - - :return: a unit instance - :rtype: pulp.plugins.model.Unit - """ - model = DockerImage(unit_dict['image_id'], unit_dict.get('parent_id'), - unit_dict.get('size')) - return self.get_conduit().init_unit(model.TYPE_ID, model.unit_key, model.unit_metadata, - model.relative_path) - - -class SaveUnits(PluginStep): - def __init__(self, working_dir): - """ - :param working_dir: full path to the directory into which image files - are downloaded. This directory should contain one - directory for each docker image, with the ID of the - docker image as its name. - :type working_dir: basestring - """ - super(SaveUnits, self).__init__(step_type=constants.SYNC_STEP_SAVE, - plugin_type=constants.IMPORTER_TYPE_ID, - working_dir=working_dir) - self.description = _('Saving images and tags') - - def process_main(self): - """ - Gets an iterable of units that were downloaded from the parent step, - moves their files into permanent storage, and then saves the unit into - the database and into the repository. - """ - _logger.debug(self.description) - for unit_key in self.parent.step_get_local_units.units_to_download: - image_id = unit_key['image_id'] - with open(os.path.join(self.working_dir, image_id, 'json')) as json_file: - metadata = json.load(json_file) - # at least one old docker image did not have a size specified in - # its metadata - size = metadata.get('Size') - # an older version of docker used a lowercase "p" - parent = metadata.get('parent', metadata.get('Parent')) - model = DockerImage(image_id, parent, size) - unit = self.get_conduit().init_unit(model.TYPE_ID, model.unit_key, model.unit_metadata, - model.relative_path) - - self.move_files(unit) - _logger.debug('saving image %s' % image_id) - self.get_conduit().save_unit(unit) - - _logger.debug('updating tags for repo %s' % self.get_repo().id) - tags.update_tags(self.get_repo().id, self.parent.tags) - - def move_files(self, unit): - """ - For the given unit, move all of its associated files from the working - directory to their permanent location. - - :param unit: a pulp unit - :type unit: pulp.plugins.model.Unit - """ - image_id = unit.unit_key['image_id'] - _logger.debug('moving files in to place for image %s' % image_id) - source_dir = os.path.join(self.working_dir, image_id) - try: - os.makedirs(unit.storage_path, mode=0755) - except OSError, e: - # it's ok if the directory exists - if e.errno != errno.EEXIST: - _logger.error('could not make directory %s' % unit.storage_path) - raise - - for name in ('json', 'ancestry', 'layer'): - shutil.move(os.path.join(source_dir, name), os.path.join(unit.storage_path, name)) diff --git a/plugins/pulp_docker/plugins/importers/tags.py b/plugins/pulp_docker/plugins/importers/tags.py deleted file mode 100644 index 08acf098..00000000 --- a/plugins/pulp_docker/plugins/importers/tags.py +++ /dev/null @@ -1,17 +0,0 @@ -from pulp.server.managers import factory -from pulp_docker.common import tags - - -def update_tags(repo_id, new_tags): - """ - Gets the current scratchpad's tags and updates them with the new_tags - - :param repo_id: unique ID of a repository - :type repo_id: basestring - :param new_tags: dictionary of tag:image_id - :type new_tags: dict - """ - repo_manager = factory.repo_manager() - scratchpad = repo_manager.get_repo_scratchpad(repo_id) - new_tags = tags.generate_updated_tags(scratchpad, new_tags) - repo_manager.update_repo_scratchpad(repo_id, {'tags': new_tags}) diff --git a/plugins/pulp_docker/plugins/importers/upload.py b/plugins/pulp_docker/plugins/importers/upload.py deleted file mode 100644 index 6ebda2f3..00000000 --- a/plugins/pulp_docker/plugins/importers/upload.py +++ /dev/null @@ -1,112 +0,0 @@ -import contextlib -import functools -import gzip -import json -import os -import tarfile - -from pulp_docker.common import models, tarutils -from pulp_docker.plugins.importers import tags - - -def get_models(metadata, mask_id=''): - """ - Given image metadata, returns model instances to represent - each layer of the image defined by the unit_key - - :param metadata: a dictionary where keys are image IDs, and values are - dictionaries with keys "parent" and "size", containing - values for those two attributes as taken from the docker - image metadata. - :type metadata: dict - :param mask_id: The ID of an image that should not be included in the - returned models. This image and all of its ancestors - will be excluded. - :type mask_id: basestring - - :return: list of models.DockerImage instances - :rtype: list - """ - images = [] - existing_image_ids = set() - - leaf_image_ids = tarutils.get_youngest_children(metadata) - - for image_id in leaf_image_ids: - while image_id: - json_data = metadata[image_id] - parent_id = json_data.get('parent') - size = json_data['size'] - - if image_id not in existing_image_ids: - # This will avoid adding multiple images with a same id, which can happen - # in case of parents with multiple children. - existing_image_ids.add(image_id) - images.append(models.DockerImage(image_id, parent_id, size)) - - if parent_id == mask_id: - break - - image_id = parent_id - - return images - - -def save_models(conduit, models, ancestry, tarfile_path): - """ - Given a collection of models, save them to pulp as Units. - - :param conduit: the conduit provided by pulp - :type conduit: pulp.plugins.conduits.unit_add.UnitAddConduit - :param models: collection of models.DockerImage instances to save - :type models: list - :param ancestry: a tuple of image IDs where the first is the image_id - passed in, and each successive ID is the parent image of - the ID that proceeds it. - :type ancestry: tuple - :param tarfile_path: full path to a tarfile that is the product - of "docker save" - :type tarfile_path: basestring - """ - with contextlib.closing(tarfile.open(tarfile_path)) as archive: - for i, model in enumerate(models): - unit = conduit.init_unit(model.TYPE_ID, model.unit_key, - model.unit_metadata, model.relative_path) - - # skip saving files if they already exist, which could happen if the - # unit already existed in pulp - if not os.path.exists(unit.storage_path): - os.makedirs(unit.storage_path, 0755) - - # save ancestry file - json.dump(ancestry[i:], open(os.path.join(unit.storage_path, 'ancestry'), 'w')) - # save json file - json_src_path = os.path.join(model.image_id, 'json') - with open(os.path.join(unit.storage_path, 'json'), 'w') as json_dest: - json_dest.write(archive.extractfile(json_src_path).read()) - # save layer file - layer_src_path = os.path.join(model.image_id, 'layer.tar') - layer_dest_path = os.path.join(unit.storage_path, 'layer') - with contextlib.closing(archive.extractfile(layer_src_path)) as layer_src: - with contextlib.closing(gzip.open(layer_dest_path, 'w')) as layer_dest: - # these can be big files, so we chunk them - reader = functools.partial(layer_src.read, 4096) - for chunk in iter(reader, ''): - layer_dest.write(chunk) - - conduit.save_unit(unit) - - -def update_tags(repo_id, tarfile_path): - """ - Gets the current scratchpad's tags and updates them with the tags contained - in the tarfile. - - :param repo_id: unique ID of a repository - :type repo_id: basestring - :param tarfile_path: full path to a tarfile that is the product - of "docker save" - :type tarfile_path: basestring - """ - new_tags = tarutils.get_tags(tarfile_path) - tags.update_tags(repo_id, new_tags) diff --git a/plugins/pulp_docker/plugins/registry.py b/plugins/pulp_docker/plugins/registry.py deleted file mode 100644 index cf6810cf..00000000 --- a/plugins/pulp_docker/plugins/registry.py +++ /dev/null @@ -1,215 +0,0 @@ -from cStringIO import StringIO -import errno -import json -import logging -import os -import urlparse - -from nectar.downloaders.threaded import HTTPThreadedDownloader -from nectar.listener import AggregatingEventListener -from nectar.request import DownloadRequest - - -_logger = logging.getLogger(__name__) - - -class Repository(object): - IMAGES_PATH = '/v1/repositories/%s/images' - TAGS_PATH = '/v1/repositories/%s/tags' - ANCESTRY_PATH = '/v1/images/%s/ancestry' - - DOCKER_TOKEN_HEADER = 'x-docker-token' - DOCKER_ENDPOINT_HEADER = 'x-docker-endpoints' - - def __init__(self, name, download_config, registry_url, working_dir): - """ - :param name: name of a docker repository - :type name: basestring - :param download_config: download configuration object - :type download_config: nectar.config.DownloaderConfig - :param registry_url: URL for the docker registry - :type registry_url: basestring - :param working_dir: full path to the directory where files should - be saved - :type working_dir: basestring - """ - self.name = name - self.download_config = download_config - self.registry_url = registry_url - self.listener = AggregatingEventListener() - self.downloader = HTTPThreadedDownloader(self.download_config, self.listener) - self.working_dir = working_dir - self.token = None - self.endpoint = None - - def _get_single_path(self, path): - """ - Retrieve a single path within the upstream registry, and return its - body after deserializing it as json - - :param path: a full http path to retrieve that will be urljoin'd to the - upstream registry url. - :type path: basestring - - :return: whatever gets deserialized out of the response body's json - """ - url = urlparse.urljoin(self.registry_url, path) - request = DownloadRequest(url, StringIO()) - if path.endswith('/images'): - # this is required by the docker index and indicates that it should - # return an auth token - if request.headers is None: - request.headers = {} - request.headers[self.DOCKER_TOKEN_HEADER] = 'true' - report = self.downloader.download_one(request) - - if report.state == report.DOWNLOAD_FAILED: - raise IOError(report.error_msg) - - self._parse_response_headers(report.headers) - return json.loads(report.destination.getvalue()) - - def _parse_response_headers(self, headers): - """ - Some responses can include header information that we need later. This - grabs those values and stores them for later use. - - :param headers: dictionary-like object where keys are HTTP header names - and values are their values. - :type headers: dict - """ - # this is used for authorization on an endpoint - if self.DOCKER_TOKEN_HEADER in headers: - self.token = headers[self.DOCKER_TOKEN_HEADER] - # this tells us what host to use when accessing image files - if self.DOCKER_ENDPOINT_HEADER in headers: - self.endpoint = headers[self.DOCKER_ENDPOINT_HEADER] - - def get_image_ids(self): - """ - Get a list of all images in the upstream repository. This is - conceptually a little ambiguous, as there can be images in a repo that - are neither tagged nor in the ancestry for a tagged image. - - :return: list of image IDs in the repo - :rtype: list - """ - path = self.IMAGES_PATH % self.name - - _logger.debug('retrieving image ids from remote registry') - raw_data = self._get_single_path(path) - return [item['id'] for item in raw_data] - - def get_tags(self): - """ - Get a dictionary of tags from the upstream repo. - - :return: a dictionary where keys are tag names, and values are either - full image IDs or abbreviated image IDs. - :rtype: dict - """ - repo_name = self.name - # this is a quirk of the docker registry API. - if '/' not in repo_name: - repo_name = 'library/' + repo_name - - path = self.TAGS_PATH % repo_name - - _logger.debug('retrieving tags from remote registry') - raw_data = self._get_single_path(path) - # raw_data will sometimes be a list of dicts, and sometimes just a dict, - # depending on what version of the API we're talking to. - if isinstance(raw_data, list): - return dict((tag['name'], tag['layer']) for tag in raw_data) - return raw_data - - def get_ancestry(self, image_ids): - """ - Retrieve the "ancestry" file for each provided image ID, and save each - in a directory whose name is the image ID. - - :param image_ids: list of image IDs for which the ancestry file - should be retrieved - :type image_ids: list - - :raises IOError: if a download fails - """ - requests = [] - for image_id in image_ids: - path = self.ANCESTRY_PATH % image_id - url = urlparse.urljoin(self.get_image_url(), path) - destination = os.path.join(self.working_dir, image_id, 'ancestry') - try: - os.mkdir(os.path.split(destination)[0]) - except OSError, e: - # it's ok if the directory already exists - if e.errno != errno.EEXIST: - raise - request = DownloadRequest(url, destination) - self.add_auth_header(request) - requests.append(request) - - _logger.debug('retrieving ancestry files from remote registry') - self.downloader.download(requests) - if len(self.listener.failed_reports): - raise IOError(self.listener.failed_reports[0].error_msg) - - def create_download_request(self, image_id, file_name, destination_dir): - """ - Return a DownloadRequest instance for the given file name and image ID. - It is desirable to download the actual layer files with a separate - downloader (for progress tracking, etc), so we just create the download - requests here and let them get processed elsewhere. - - This adds the Authorization header if a token is known for this - repository. - - :param image_id: unique ID of a docker image - :type image_id: basestring - :param file_name: name of the file, one of "ancestry", "json", - or "layer" - :type file_name: basestring - :param destination_dir: full path to the directory where file should - be saved - :type destination_dir: basestring - - :return: a download request instance - :rtype: nectar.request.DownloadRequest - """ - url = self.get_image_url() - req = DownloadRequest(urlparse.urljoin(url, '/v1/images/%s/%s' % (image_id, file_name)), - os.path.join(destination_dir, file_name)) - self.add_auth_header(req) - return req - - def add_auth_header(self, request): - """ - Given a download request, add an Authorization header if we have an - auth token available. - - :param request: a download request - :type request: nectar.request.DownloadRequest - """ - if self.token: - if request.headers is None: - request.headers = {} - # this emulates what docker itself does - request.headers['Authorization'] = 'Token %s' % self.token - - def get_image_url(self): - """ - Get a URL for the registry or the endpoint, for use in retrieving image - files. The "endpoint" is a host name that might be returned in a header - when retrieving repository data above. - - :return: a url that is either the provided registry url, or if an - endpoint is known, that same url with the host replaced by - the endpoint - :rtype: basestring - """ - if self.endpoint: - parts = list(urlparse.urlsplit(self.registry_url)) - parts[1] = self.endpoint - return urlparse.urlunsplit(parts) - else: - return self.registry_url diff --git a/plugins/setup.py b/plugins/setup.py deleted file mode 100644 index 9b732fdc..00000000 --- a/plugins/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_docker_plugins', - version='0.1.0', - packages=find_packages(), - url='http://www.pulpproject.org', - license='GPLv2+', - author='Pulp Team', - author_email='pulp-list@redhat.com', - description='plugins for docker image support in pulp', - entry_points={ - 'pulp.importers': [ - 'importer = pulp_docker.plugins.importers.importer:entry_point', - ], - 'pulp.distributors': [ - 'web_distributor = pulp_docker.plugins.distributors.distributor_web:entry_point', - 'export_distributor = pulp_docker.plugins.distributors.distributor_export:entry_point', - ] - } -) diff --git a/plugins/test/__init__.py b/plugins/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/test/data/busyboxlight.tar b/plugins/test/data/busyboxlight.tar deleted file mode 100644 index 3ad09df1..00000000 Binary files a/plugins/test/data/busyboxlight.tar and /dev/null differ diff --git a/plugins/test/unit/__init__.py b/plugins/test/unit/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/test/unit/plugins/__init__.py b/plugins/test/unit/plugins/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/test/unit/plugins/distributors/__init__.py b/plugins/test/unit/plugins/distributors/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/test/unit/plugins/distributors/test_configuration.py b/plugins/test/unit/plugins/distributors/test_configuration.py deleted file mode 100644 index fb6203d9..00000000 --- a/plugins/test/unit/plugins/distributors/test_configuration.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -from mock import Mock, patch - -from pulp.devel.unit.server.util import assert_validation_exception -from pulp.plugins.config import PluginCallConfiguration - -from pulp_docker.common import constants, error_codes -from pulp_docker.plugins.distributors import configuration - - -class TestValidateConfig(unittest.TestCase): - - def test_server_url_fully_qualified(self): - config = { - constants.CONFIG_KEY_REDIRECT_URL: 'http://www.pulpproject.org/foo' - } - self.assertEquals((True, None), configuration.validate_config(config)) - - def test_server_url_fully_qualified_with_port(self): - config = { - constants.CONFIG_KEY_REDIRECT_URL: 'http://www.pulpproject.org:440/foo' - } - self.assertEquals((True, None), configuration.validate_config(config)) - - def test_server_url_empty(self): - config = { - constants.CONFIG_KEY_REDIRECT_URL: '' - } - # This is valid as the default server should be used - - self.assertEquals((True, None), configuration.validate_config(config)) - - def test_server_url_missing_host_and_path(self): - config = { - constants.CONFIG_KEY_REDIRECT_URL: 'http://' - } - assert_validation_exception(configuration.validate_config, - [error_codes.DKR1002, - error_codes.DKR1003], config) - - def test_server_url_missing_scheme(self): - config = { - constants.CONFIG_KEY_REDIRECT_URL: 'www.pulpproject.org/foo' - } - assert_validation_exception(configuration.validate_config, - [error_codes.DKR1001, - error_codes.DKR1002], config) - - def test_configuration_protected_true(self): - config = PluginCallConfiguration({ - constants.CONFIG_KEY_PROTECTED: True - }, {}) - - self.assertEquals((True, None), configuration.validate_config(config)) - - def test_configuration_protected_false_str(self): - config = PluginCallConfiguration({ - constants.CONFIG_KEY_PROTECTED: 'false' - }, {}) - - self.assertEquals((True, None), configuration.validate_config(config)) - - def test_configuration_protected_bad_str(self): - config = PluginCallConfiguration({ - constants.CONFIG_KEY_PROTECTED: 'apple' - }, {}) - assert_validation_exception(configuration.validate_config, - [error_codes.DKR1004], config) - - -class TestConfigurationGetters(unittest.TestCase): - - def setUp(self): - self.working_directory = tempfile.mkdtemp() - self.publish_dir = os.path.join(self.working_directory, 'publish') - self.repo_working = os.path.join(self.working_directory, 'work') - - self.repo = Mock(id='foo', working_dir=self.repo_working) - self.config = { - constants.CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY: self.publish_dir - } - - def tearDown(self): - shutil.rmtree(self.working_directory) - - def test_get_root_publish_directory(self): - directory = configuration.get_root_publish_directory(self.config) - self.assertEquals(directory, self.publish_dir) - - def test_get_master_publish_dir(self): - directory = configuration.get_master_publish_dir(self.repo, self.config) - self.assertEquals(directory, os.path.join(self.publish_dir, 'master', self.repo.id)) - - def test_get_web_publish_dir(self): - directory = configuration.get_web_publish_dir(self.repo, self.config) - self.assertEquals(directory, os.path.join(self.publish_dir, 'web', self.repo.id)) - - def test_get_repo_relative_path(self): - directory = configuration.get_repo_relative_path(self.repo, self.config) - self.assertEquals(directory, self.repo.id) - - def test_get_redirect_url_from_config(self): - sample_url = 'http://www.pulpproject.org/' - conduit = Mock(repo_id=sample_url) - url = configuration.get_redirect_url({constants.CONFIG_KEY_REDIRECT_URL: sample_url}, - conduit) - self.assertEquals(sample_url, url) - - def test_get_redirect_url_from_config_trailing_slash(self): - sample_url = 'http://www.pulpproject.org' - conduit = Mock(repo_id=sample_url) - url = configuration.get_redirect_url({constants.CONFIG_KEY_REDIRECT_URL: sample_url}, - conduit) - self.assertEquals(sample_url + '/', url) - - @patch('pulp_docker.plugins.distributors.configuration.server_config') - def test_get_redirect_url_generated(self, mock_server_config): - mock_server_config.get.return_value = 'www.foo.bar' - computed_result = 'https://www.foo.bar/pulp/docker/baz/' - self.assertEquals(computed_result, configuration.get_redirect_url({}, - Mock(id='baz'))) - - def test_get_export_repo_filename(self): - filename = configuration.get_export_repo_filename(self.repo, self.config) - self.assertEquals(filename, "foo.tar") - - def test_get_export_repo_directory(self): - directory = configuration.get_export_repo_directory(self.config) - self.assertEquals(directory, os.path.join(self.publish_dir, 'export', 'repo')) - - def test_get_export_repo_file_with_path_from_config(self): - config = PluginCallConfiguration(None, {constants.CONFIG_KEY_EXPORT_FILE: '/tmp/foo.tar'}) - result = configuration.get_export_repo_file_with_path(self.repo, config) - self.assertEquals(result, '/tmp/foo.tar') - - def test_get_export_repo_file_with_path_default(self): - result = configuration.get_export_repo_file_with_path(self.repo, self.config) - expected_result = os.path.join(configuration.get_export_repo_directory(self.config), - configuration.get_export_repo_filename(self.repo, - self.config)) - self.assertEquals(result, expected_result) diff --git a/plugins/test/unit/plugins/distributors/test_distributor_export.py b/plugins/test/unit/plugins/distributors/test_distributor_export.py deleted file mode 100644 index 9b95d9a6..00000000 --- a/plugins/test/unit/plugins/distributors/test_distributor_export.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -from mock import Mock, MagicMock, patch -from pulp.devel.unit.util import touch -from pulp.plugins.conduits.repo_publish import RepoPublishConduit -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.model import Repository -from pulp.plugins.distributor import Distributor - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors.distributor_export import DockerExportDistributor, entry_point - - -class TestEntryPoint(unittest.TestCase): - def test_returns_importer(self): - distributor, config = entry_point() - - self.assertTrue(issubclass(distributor, Distributor)) - - def test_returns_config(self): - distributor, config = entry_point() - - # make sure it's at least the correct type - self.assertTrue(isinstance(config, dict)) - - -class TestBasics(unittest.TestCase): - - def setUp(self): - self.distributor = DockerExportDistributor() - self.working_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.working_dir, ignore_errors=True) - - def test_metadata(self): - metadata = DockerExportDistributor.metadata() - - self.assertEqual(metadata['id'], constants.DISTRIBUTOR_EXPORT_TYPE_ID) - self.assertEqual(metadata['types'], [constants.IMAGE_TYPE_ID]) - self.assertTrue(len(metadata['display_name']) > 0) - - @patch('pulp_docker.plugins.distributors.distributor_export.configuration.validate_config') - def test_validate_config(self, mock_validate): - value = self.distributor.validate_config(Mock(), 'foo', Mock()) - mock_validate.assert_called_once_with('foo') - self.assertEquals(value, mock_validate.return_value) - - @patch('pulp_docker.plugins.distributors.distributor_export.configuration.' - 'get_export_repo_directory') - def test_distributor_removed(self, mock_repo_dir): - mock_repo_dir.return_value = os.path.join(self.working_dir, 'repo') - os.makedirs(mock_repo_dir.return_value) - working_dir = os.path.join(self.working_dir, 'working') - repo = Mock(id='bar', working_dir=working_dir) - config = {} - touch(os.path.join(working_dir, 'bar.json')) - touch(os.path.join(mock_repo_dir.return_value, 'bar.tar')) - self.distributor.distributor_removed(repo, config) - - self.assertEquals(0, len(os.listdir(mock_repo_dir.return_value))) - self.assertEquals(1, len(os.listdir(self.working_dir))) - - @patch('pulp_docker.plugins.distributors.distributor_export.configuration.' - 'get_export_repo_directory') - def test_distributor_removed_files_missing(self, mock_repo_dir): - mock_repo_dir.return_value = os.path.join(self.working_dir, 'repo') - os.makedirs(mock_repo_dir.return_value) - working_dir = os.path.join(self.working_dir, 'working') - repo = Mock(id='bar', working_dir=working_dir) - config = {} - self.distributor.distributor_removed(repo, config) - - self.assertEquals(1, len(os.listdir(self.working_dir))) - self.assertEquals(0, len(os.listdir(mock_repo_dir.return_value))) - - @patch('pulp_docker.plugins.distributors.distributor_export.ExportPublisher') - def test_publish_repo(self, mock_publisher): - repo = Repository('test') - config = PluginCallConfiguration(None, None) - conduit = RepoPublishConduit(repo.id, 'foo_repo') - self.distributor.publish_repo(repo, conduit, config) - - mock_publisher.return_value.assert_called_once() - - def test_cancel_publish_repo(self): - self.distributor._publisher = MagicMock() - self.distributor.cancel_publish_repo() - self.assertTrue(self.distributor.canceled) - - self.distributor._publisher.cancel.assert_called_once() diff --git a/plugins/test/unit/plugins/distributors/test_distributor_web.py b/plugins/test/unit/plugins/distributors/test_distributor_web.py deleted file mode 100644 index 2c863fec..00000000 --- a/plugins/test/unit/plugins/distributors/test_distributor_web.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -from mock import Mock, MagicMock, patch -from pulp.devel.unit.util import touch -from pulp.plugins.conduits.repo_publish import RepoPublishConduit -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.model import Repository -from pulp.plugins.distributor import Distributor - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors.distributor_web import DockerWebDistributor, entry_point - - -class TestEntryPoint(unittest.TestCase): - def test_returns_importer(self): - distributor, config = entry_point() - - self.assertTrue(issubclass(distributor, Distributor)) - - def test_returns_config(self): - distributor, config = entry_point() - - # make sure it's at least the correct type - self.assertTrue(isinstance(config, dict)) - - -class TestBasics(unittest.TestCase): - - def setUp(self): - self.distributor = DockerWebDistributor() - self.working_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.working_dir) - - def test_metadata(self): - metadata = DockerWebDistributor.metadata() - - self.assertEqual(metadata['id'], constants.DISTRIBUTOR_WEB_TYPE_ID) - self.assertEqual(metadata['types'], [constants.IMAGE_TYPE_ID]) - self.assertTrue(len(metadata['display_name']) > 0) - - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.validate_config') - def test_validate_config(self, mock_validate): - value = self.distributor.validate_config(Mock(), 'foo', Mock()) - mock_validate.assert_called_once_with('foo') - self.assertEquals(value, mock_validate.return_value) - - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.get_app_publish_dir') - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.get_master_publish_dir') - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.get_web_publish_dir') - def test_distributor_removed(self, mock_web, mock_master, mock_app): - - mock_app.return_value = os.path.join(self.working_dir) - mock_web.return_value = os.path.join(self.working_dir, 'web') - mock_master.return_value = os.path.join(self.working_dir, 'master') - working_dir = os.path.join(self.working_dir, 'working') - os.makedirs(mock_web.return_value) - os.makedirs(mock_master.return_value) - repo = Mock(id='bar', working_dir=working_dir) - config = {} - touch(os.path.join(self.working_dir, 'bar.json')) - self.distributor.distributor_removed(repo, config) - - self.assertEquals(0, len(os.listdir(self.working_dir))) - - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.get_app_publish_dir') - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.get_master_publish_dir') - @patch('pulp_docker.plugins.distributors.distributor_web.configuration.get_web_publish_dir') - def test_distributor_removed_files_missing(self, mock_web, mock_master, mock_app): - mock_app.return_value = os.path.join(self.working_dir) - mock_web.return_value = os.path.join(self.working_dir, 'web') - mock_master.return_value = os.path.join(self.working_dir, 'master') - working_dir = os.path.join(self.working_dir, 'working') - repo = Mock(id='bar', working_dir=working_dir) - config = {} - self.distributor.distributor_removed(repo, config) - self.assertEquals(0, len(os.listdir(self.working_dir))) - - @patch('pulp_docker.plugins.distributors.distributor_web.WebPublisher') - def test_publish_repo(self, mock_publisher): - repo = Repository('test') - config = PluginCallConfiguration(None, None) - conduit = RepoPublishConduit(repo.id, 'foo_repo') - self.distributor.publish_repo(repo, conduit, config) - - mock_publisher.return_value.assert_called_once() - - def test_cancel_publish_repo(self): - self.distributor._publisher = MagicMock() - self.distributor.cancel_publish_repo() - self.assertTrue(self.distributor.canceled) - - self.distributor._publisher.cancel.assert_called_once() diff --git a/plugins/test/unit/plugins/distributors/test_metadata.py b/plugins/test/unit/plugins/distributors/test_metadata.py deleted file mode 100644 index 819a2b4b..00000000 --- a/plugins/test/unit/plugins/distributors/test_metadata.py +++ /dev/null @@ -1,94 +0,0 @@ -import shutil -import tempfile -import unittest - -from mock import Mock, call -from pulp.common.compat import json -from pulp.plugins.conduits.repo_publish import RepoPublishConduit -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.model import Repository - -from pulp_docker.common import constants -from pulp_docker.common.models import DockerImage -from pulp_docker.plugins.distributors import metadata - - -class TestRedirectFileContext(unittest.TestCase): - - def setUp(self): - self.working_directory = tempfile.mkdtemp() - self.repo = Repository('foo_repo_id', working_dir=self.working_directory) - self.config = PluginCallConfiguration(None, None) - self.conduit = RepoPublishConduit(self.repo.id, 'foo_repo') - self.conduit.get_repo_scratchpad = Mock(return_value={u'tags': []}) - tag_list = [{constants.IMAGE_TAG_KEY: u'latest', - constants.IMAGE_ID_KEY: u'image_id'}] - self.conduit.get_repo_scratchpad.return_value = {u'tags': tag_list} - self.context = metadata.RedirectFileContext(self.working_directory, - self.conduit, - self.config, - self.repo) - self.context.metadata_file_handle = Mock() - - def tearDown(self): - shutil.rmtree(self.working_directory) - - def test_add_unit_metadata(self): - unit = DockerImage('foo_image', 'foo_parent', 2048) - test_result = {'id': 'foo_image'} - result_json = json.dumps(test_result) - self.context.add_unit_metadata(unit) - self.context.metadata_file_handle.write.assert_called_once_with(result_json) - - def test_add_unit_metadata_with_tag(self): - unit = DockerImage('foo_image', 'foo_parent', 2048) - test_result = {'id': 'foo_image'} - result_json = json.dumps(test_result) - self.context.tags = {'bar': 'foo_image'} - self.context.redirect_url = 'http://www.pulpproject.org/foo/' - self.context.add_unit_metadata(unit) - self.context.metadata_file_handle.write.assert_called_once_with(result_json) - - def test_write_file_header(self): - self.context.repo_id = 'bar' - self.context.redirect_url = 'http://www.pulpproject.org/foo/' - - self.context._write_file_header() - result_string = '{"type":"pulp-docker-redirect","version":1,"repository":"bar",' \ - '"repo-registry-id": "foo_repo_id",' \ - '"url":"http://www.pulpproject.org/foo/",' \ - '"protected":false,"images":[' - self.context.metadata_file_handle.write.assert_called_once_with(result_string) - - def test_write_file_protected_true(self): - self.config = PluginCallConfiguration(None, {'protected': True}) - self.context = metadata.RedirectFileContext(self.working_directory, - self.conduit, - self.config, - self.repo) - self.context.metadata_file_handle = Mock() - self.context.repo_id = 'bar' - self.context.redirect_url = 'http://www.pulpproject.org/foo/' - - self.context._write_file_header() - result_string = '{"type":"pulp-docker-redirect","version":1,"repository":"bar",' \ - '"repo-registry-id": "foo_repo_id",' \ - '"url":"http://www.pulpproject.org/foo/",' \ - '"protected":true,"images":[' - self.context.metadata_file_handle.write.assert_called_once_with(result_string) - - def test_write_file_footer(self): - self.context._write_file_footer() - calls = [call('],"tags":'), call(json.dumps({u'latest': u'image_id'})), call('}')] - - self.context.metadata_file_handle.write.assert_has_calls(calls) - - def test_convert_tag_list_to_dict(self): - self.assertEqual(self.context.convert_tag_list_to_dict([]), {}) - tag_list = [{constants.IMAGE_TAG_KEY: 'tag1', - constants.IMAGE_ID_KEY: 'image1'}, - {constants.IMAGE_TAG_KEY: 'tag2', - constants.IMAGE_ID_KEY: 'image2'}] - tag_dict = self.context.convert_tag_list_to_dict(tag_list) - expected_tag_dict = {'tag1': 'image1', 'tag2': 'image2'} - self.assertEqual(tag_dict, expected_tag_dict) diff --git a/plugins/test/unit/plugins/distributors/test_publish_steps.py b/plugins/test/unit/plugins/distributors/test_publish_steps.py deleted file mode 100644 index a35aa1a0..00000000 --- a/plugins/test/unit/plugins/distributors/test_publish_steps.py +++ /dev/null @@ -1,111 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -from mock import Mock, patch - -from pulp.devel.unit.util import touch -from pulp.plugins.conduits.repo_publish import RepoPublishConduit -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.model import Repository -from pulp.plugins.util.publish_step import PublishStep - -from pulp_docker.common import constants -from pulp_docker.plugins.distributors import publish_steps - - -class TestPublishImagesStep(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.mkdtemp() - self.working_directory = os.path.join(self.temp_dir, 'working') - self.publish_directory = os.path.join(self.temp_dir, 'publish') - self.content_directory = os.path.join(self.temp_dir, 'content') - os.makedirs(self.working_directory) - os.makedirs(self.publish_directory) - os.makedirs(self.content_directory) - repo = Repository('foo_repo_id', working_dir=self.working_directory) - config = PluginCallConfiguration(None, None) - conduit = RepoPublishConduit(repo.id, 'foo_repo') - conduit.get_repo_scratchpad = Mock(return_value={u'tags': {}}) - self.parent = PublishStep('test-step', repo, conduit, config) - - def tearDown(self): - shutil.rmtree(self.temp_dir) - - @patch('pulp_docker.plugins.distributors.publish_steps.RedirectFileContext') - def test_initialize_metdata(self, mock_context): - step = publish_steps.PublishImagesStep() - step.parent = self.parent - step.initialize() - mock_context.return_value.initialize.assert_called_once_with() - - def test_process_units(self): - step = publish_steps.PublishImagesStep() - step.parent = self.parent - step.redirect_context = Mock() - file_list = ['ancestry', 'layer', 'json'] - for file_name in file_list: - touch(os.path.join(self.content_directory, file_name)) - unit = Mock(unit_key={'image_id': 'foo_image'}, storage_path=self.content_directory) - step.get_working_dir = Mock(return_value=self.publish_directory) - step.process_unit(unit) - step.redirect_context.add_unit_metadata.assert_called_once_with(unit) - for file_name in file_list: - self.assertTrue(os.path.exists(os.path.join(self.publish_directory, 'web', - 'foo_image', file_name))) - - def test_finalize(self): - step = publish_steps.PublishImagesStep() - step.redirect_context = Mock() - step.finalize() - step.redirect_context.finalize.assert_called_once_with() - - -class TestWebPublisher(unittest.TestCase): - - def setUp(self): - self.working_directory = tempfile.mkdtemp() - self.publish_dir = os.path.join(self.working_directory, 'publish') - self.master_dir = os.path.join(self.working_directory, 'master') - self.working_temp = os.path.join(self.working_directory, 'work') - self.repo = Mock(id='foo', working_dir=self.working_temp) - - def tearDown(self): - shutil.rmtree(self.working_directory) - - @patch('pulp_docker.plugins.distributors.publish_steps.AtomicDirectoryPublishStep') - @patch('pulp_docker.plugins.distributors.publish_steps.PublishImagesStep') - def test_init(self, mock_images_step, mock_web_publish_step): - mock_conduit = Mock() - mock_config = { - constants.CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY: self.publish_dir - } - publisher = publish_steps.WebPublisher(self.repo, mock_conduit, mock_config) - self.assertEquals(publisher.children, [mock_images_step.return_value, - mock_web_publish_step.return_value]) - - -class TestExportPublisher(unittest.TestCase): - - def setUp(self): - self.working_directory = tempfile.mkdtemp() - self.publish_dir = os.path.join(self.working_directory, 'publish') - self.working_temp = os.path.join(self.working_directory, 'work') - self.repo = Mock(id='foo', working_dir=self.working_temp) - - def tearDown(self): - shutil.rmtree(self.working_directory) - - def test_init(self): - mock_conduit = Mock() - mock_config = { - constants.CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY: self.publish_dir - } - publisher = publish_steps.ExportPublisher(self.repo, mock_conduit, mock_config) - self.assertTrue(isinstance(publisher.children[0], publish_steps.PublishImagesStep)) - self.assertTrue(isinstance(publisher.children[1], publish_steps.SaveTarFilePublishStep)) - tar_step = publisher.children[1] - self.assertEquals(tar_step.source_dir, self.working_temp) - self.assertEquals(tar_step.publish_file, - os.path.join(self.publish_dir, 'export/repo/foo.tar')) diff --git a/plugins/test/unit/plugins/importers/__init__.py b/plugins/test/unit/plugins/importers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/plugins/test/unit/plugins/importers/data.py b/plugins/test/unit/plugins/importers/data.py deleted file mode 100644 index 3f2a7297..00000000 --- a/plugins/test/unit/plugins/importers/data.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - - -busybox_tar_path = os.path.join(os.path.dirname(__file__), '../../../data/busyboxlight.tar') - -# these are in correct ancestry order -busybox_ids = ( - '769b9341d937a3dba9e460f664b4f183a6cecdd62b337220a28b3deb50ee0a02', - '48e5f45168b97799ad0aafb7e2fef9fac57b5f16f6db7f67ba2000eb947637eb', - 'bf747efa0e2fa9f7c691588ce3938944c75607a7bb5e757f7369f86904d97c78', - '511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158', -) diff --git a/plugins/test/unit/plugins/importers/test_importer.py b/plugins/test/unit/plugins/importers/test_importer.py deleted file mode 100644 index b53cd008..00000000 --- a/plugins/test/unit/plugins/importers/test_importer.py +++ /dev/null @@ -1,219 +0,0 @@ -import unittest - -import mock -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.importer import Importer -from pulp.plugins.model import Repository - -import data -from pulp_docker.common import constants -from pulp_docker.common.models import DockerImage -from pulp_docker.plugins.importers.importer import DockerImporter, entry_point -from pulp_docker.plugins.importers import upload - - -class TestEntryPoint(unittest.TestCase): - def test_returns_importer(self): - importer, config = entry_point() - - self.assertTrue(issubclass(importer, Importer)) - - def test_returns_config(self): - importer, config = entry_point() - - # make sure it's at least the correct type - self.assertTrue(isinstance(config, dict)) - - -class TestBasics(unittest.TestCase): - def test_metadata(self): - metadata = DockerImporter.metadata() - - self.assertEqual(metadata['id'], constants.IMPORTER_TYPE_ID) - self.assertEqual(metadata['types'], [constants.IMAGE_TYPE_ID]) - self.assertTrue(len(metadata['display_name']) > 0) - - -@mock.patch('pulp_docker.plugins.importers.sync.SyncStep') -@mock.patch('tempfile.mkdtemp', spec_set=True) -@mock.patch('shutil.rmtree') -class TestSyncRepo(unittest.TestCase): - def setUp(self): - super(TestSyncRepo, self).setUp() - self.repo = Repository('repo1', working_dir='/a/b/c') - self.sync_conduit = mock.MagicMock() - self.config = mock.MagicMock() - self.importer = DockerImporter() - - def test_calls_sync_step(self, mock_rmtree, mock_mkdtemp, mock_sync_step): - self.importer.sync_repo(self.repo, self.sync_conduit, self.config) - - mock_sync_step.assert_called_once_with(repo=self.repo, conduit=self.sync_conduit, - config=self.config, - working_dir=mock_mkdtemp.return_value) - - def test_calls_sync(self, mock_rmtree, mock_mkdtemp, mock_sync_step): - self.importer.sync_repo(self.repo, self.sync_conduit, self.config) - - mock_sync_step.return_value.sync.assert_called_once_with() - - def test_makes_temp_dir(self, mock_rmtree, mock_mkdtemp, mock_sync_step): - self.importer.sync_repo(self.repo, self.sync_conduit, self.config) - - mock_mkdtemp.assert_called_once_with(dir=self.repo.working_dir) - - def test_removes_temp_dir(self, mock_rmtree, mock_mkdtemp, mock_sync_step): - self.importer.sync_repo(self.repo, self.sync_conduit, self.config) - - mock_rmtree.assert_called_once_with(mock_mkdtemp.return_value, ignore_errors=True) - - def test_removes_temp_dir_after_exception(self, mock_rmtree, mock_mkdtemp, mock_sync_step): - class MyError(Exception): - pass - mock_sync_step.return_value.sync.side_effect = MyError - self.assertRaises(MyError, self.importer.sync_repo, self.repo, - self.sync_conduit, self.config) - - mock_rmtree.assert_called_once_with(mock_mkdtemp.return_value, ignore_errors=True) - - -class TestCancel(unittest.TestCase): - def setUp(self): - super(TestCancel, self).setUp() - self.importer = DockerImporter() - - def test_calls_cancel(self): - self.importer.sync_step = mock.MagicMock() - - self.importer.cancel_sync_repo() - - # make sure the step's cancel method was called - self.importer.sync_step.cancel.assert_called_once_with() - - -@mock.patch.object(upload, 'update_tags', spec_set=True) -class TestUploadUnit(unittest.TestCase): - def setUp(self): - self.unit_key = {'image_id': data.busybox_ids[0]} - self.repo = Repository('repo1') - self.conduit = mock.MagicMock() - self.config = PluginCallConfiguration({}, {}) - - @mock.patch('pulp_docker.plugins.importers.upload.save_models', spec_set=True) - def test_save_conduit(self, mock_save, mock_update_tags): - DockerImporter().upload_unit(self.repo, constants.IMAGE_TYPE_ID, self.unit_key, - {}, data.busybox_tar_path, self.conduit, self.config) - - conduit = mock_save.call_args[0][0] - - self.assertTrue(conduit is self.conduit) - - @mock.patch('pulp_docker.plugins.importers.upload.save_models', spec_set=True) - def test_saved_models(self, mock_save, mock_update_tags): - DockerImporter().upload_unit(self.repo, constants.IMAGE_TYPE_ID, self.unit_key, - {}, data.busybox_tar_path, self.conduit, self.config) - - models = mock_save.call_args[0][1] - - for model in models: - self.assertTrue(isinstance(model, DockerImage)) - - ids = [m.image_id for m in models] - - self.assertEqual(tuple(ids), data.busybox_ids) - - @mock.patch('pulp_docker.plugins.importers.upload.save_models', spec_set=True) - def test_saved_ancestry(self, mock_save, mock_update_tags): - DockerImporter().upload_unit(self.repo, constants.IMAGE_TYPE_ID, self.unit_key, - {}, data.busybox_tar_path, self.conduit, self.config) - - ancestry = mock_save.call_args[0][2] - - self.assertEqual(tuple(ancestry), data.busybox_ids) - - @mock.patch('pulp_docker.plugins.importers.upload.save_models', spec_set=True) - def test_saved_filepath(self, mock_save, mock_update_tags): - DockerImporter().upload_unit(self.repo, constants.IMAGE_TYPE_ID, self.unit_key, - {}, data.busybox_tar_path, self.conduit, self.config) - - path = mock_save.call_args[0][3] - - self.assertEqual(path, data.busybox_tar_path) - - @mock.patch('pulp_docker.plugins.importers.upload.save_models', spec_set=True) - def test_added_tags(self, mock_save, mock_update_tags): - DockerImporter().upload_unit(self.repo, constants.IMAGE_TYPE_ID, self.unit_key, - {}, data.busybox_tar_path, self.conduit, self.config) - - mock_update_tags.assert_called_once_with(self.repo.id, data.busybox_tar_path) - - -class TestImportUnits(unittest.TestCase): - - def setUp(self): - self.unit_key = {'image_id': data.busybox_ids[0]} - self.source_repo = Repository('repo_source') - self.dest_repo = Repository('repo_dest') - self.conduit = mock.MagicMock() - self.config = PluginCallConfiguration({}, {}) - - def test_import_all(self): - mock_unit = mock.Mock(unit_key={'image_id': 'foo'}, metadata={}) - self.conduit.get_source_units.return_value = [mock_unit] - result = DockerImporter().import_units(self.source_repo, self.dest_repo, self.conduit, - self.config) - self.assertEquals(result, [mock_unit]) - self.conduit.associate_unit.assert_called_once_with(mock_unit) - - def test_import_no_parent(self): - mock_unit = mock.Mock(unit_key={'image_id': 'foo'}, metadata={}) - result = DockerImporter().import_units(self.source_repo, self.dest_repo, self.conduit, - self.config, units=[mock_unit]) - self.assertEquals(result, [mock_unit]) - self.conduit.associate_unit.assert_called_once_with(mock_unit) - - def test_import_with_parent(self): - mock_unit1 = mock.Mock(unit_key={'image_id': 'foo'}, metadata={'parent_id': 'bar'}) - mock_unit2 = mock.Mock(unit_key={'image_id': 'bar'}, metadata={}) - self.conduit.get_source_units.return_value = [mock_unit2] - result = DockerImporter().import_units(self.source_repo, self.dest_repo, self.conduit, - self.config, units=[mock_unit1]) - self.assertEquals(result, [mock_unit1, mock_unit2]) - calls = [mock.call(mock_unit1), mock.call(mock_unit2)] - self.conduit.associate_unit.assert_has_calls(calls) - - -class TestValidateConfig(unittest.TestCase): - def test_always_true(self): - for repo, config in [['a', 'b'], [1, 2], [mock.Mock(), {}], ['abc', {'a': 2}]]: - # make sure all attempts are validated - self.assertEqual(DockerImporter().validate_config(repo, config), (True, '')) - - -class TestRemoveUnit(unittest.TestCase): - - def setUp(self): - self.repo = Repository('repo_source') - self.conduit = mock.MagicMock() - self.config = PluginCallConfiguration({}, {}) - self.mock_unit = mock.Mock(unit_key={'image_id': 'foo'}, metadata={}) - - @mock.patch('pulp_docker.plugins.importers.importer.manager_factory.repo_manager') - def test_remove_with_tag(self, mock_repo_manager): - mock_repo_manager.return_value.get_repo_scratchpad.return_value = \ - {u'tags': [{constants.IMAGE_TAG_KEY: 'apple', - constants.IMAGE_ID_KEY: 'foo'}]} - DockerImporter().remove_units(self.repo, [self.mock_unit], self.config) - mock_repo_manager.return_value.set_repo_scratchpad.assert_called_once_with( - self.repo.id, {u'tags': []} - ) - - @mock.patch('pulp_docker.plugins.importers.importer.manager_factory.repo_manager') - def test_remove_without_tag(self, mock_repo_manager): - expected_tags = {u'tags': [{constants.IMAGE_TAG_KEY: 'apple', - constants.IMAGE_ID_KEY: 'bar'}]} - mock_repo_manager.return_value.get_repo_scratchpad.return_value = expected_tags - - DockerImporter().remove_units(self.repo, [self.mock_unit], self.config) - mock_repo_manager.return_value.set_repo_scratchpad.assert_called_once_with( - self.repo.id, expected_tags) diff --git a/plugins/test/unit/plugins/importers/test_sync.py b/plugins/test/unit/plugins/importers/test_sync.py deleted file mode 100644 index 2282763f..00000000 --- a/plugins/test/unit/plugins/importers/test_sync.py +++ /dev/null @@ -1,358 +0,0 @@ -import inspect -import json -import os -import shutil -import tempfile -import unittest - -import mock -from nectar.request import DownloadRequest -from pulp.common.plugins import importer_constants, reporting_constants -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.model import Repository as RepositoryModel, Unit -from pulp.server.managers import factory - -from pulp_docker.common import constants -from pulp_docker.plugins.importers import sync -from pulp_docker.plugins import registry - - -factory.initialize() - - -class TestSyncStep(unittest.TestCase): - def setUp(self): - super(TestSyncStep, self).setUp() - - self.repo = RepositoryModel('repo1') - self.conduit = mock.MagicMock() - plugin_config = { - constants.CONFIG_KEY_UPSTREAM_NAME: 'pulp/crane', - importer_constants.KEY_FEED: 'http://pulpproject.org/', - } - self.config = PluginCallConfiguration({}, plugin_config) - self.step = sync.SyncStep(self.repo, self.conduit, self.config, '/a/b/c') - - def test_init(self): - self.assertEqual(self.step.step_id, constants.SYNC_STEP_MAIN) - - # make sure the children are present - step_ids = set([child.step_id for child in self.step.children]) - self.assertTrue(constants.SYNC_STEP_METADATA in step_ids) - self.assertTrue(reporting_constants.SYNC_STEP_GET_LOCAL in step_ids) - self.assertTrue(constants.SYNC_STEP_DOWNLOAD in step_ids) - self.assertTrue(constants.SYNC_STEP_SAVE in step_ids) - - # make sure it instantiated a Repository object - self.assertTrue(isinstance(self.step.index_repository, registry.Repository)) - self.assertEqual(self.step.index_repository.name, 'pulp/crane') - self.assertEqual(self.step.index_repository.registry_url, 'http://pulpproject.org/') - - # these are important because child steps will populate them with data - self.assertEqual(self.step.available_units, []) - self.assertEqual(self.step.tags, {}) - - def test_generate_download_requests(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - self.step.working_dir = tempfile.mkdtemp() - - try: - generator = self.step.generate_download_requests() - self.assertTrue(inspect.isgenerator(generator)) - - download_reqs = list(generator) - - self.assertEqual(len(download_reqs), 3) - for req in download_reqs: - self.assertTrue(isinstance(req, DownloadRequest)) - finally: - shutil.rmtree(self.step.working_dir) - - def test_generate_download_requests_correct_urls(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - self.step.working_dir = tempfile.mkdtemp() - - try: - generator = self.step.generate_download_requests() - - # make sure the urls are correct - urls = [req.url for req in generator] - self.assertTrue('http://pulpproject.org/v1/images/image1/ancestry' in urls) - self.assertTrue('http://pulpproject.org/v1/images/image1/json' in urls) - self.assertTrue('http://pulpproject.org/v1/images/image1/layer' in urls) - finally: - shutil.rmtree(self.step.working_dir) - - def test_generate_download_requests_correct_destinations(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - self.step.working_dir = tempfile.mkdtemp() - - try: - generator = self.step.generate_download_requests() - - # make sure the urls are correct - destinations = [req.destination for req in generator] - self.assertTrue(os.path.join(self.step.working_dir, 'image1', 'ancestry') - in destinations) - self.assertTrue(os.path.join(self.step.working_dir, 'image1', 'json') - in destinations) - self.assertTrue(os.path.join(self.step.working_dir, 'image1', 'layer') - in destinations) - finally: - shutil.rmtree(self.step.working_dir) - - def test_generate_download_reqs_creates_dir(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - self.step.working_dir = tempfile.mkdtemp() - - try: - list(self.step.generate_download_requests()) - - # make sure it created the destination directory - self.assertTrue(os.path.isdir(os.path.join(self.step.working_dir, 'image1'))) - finally: - shutil.rmtree(self.step.working_dir) - - def test_generate_download_reqs_existing_dir(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - self.step.working_dir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.step.working_dir, 'image1')) - - try: - # just make sure this doesn't complain - list(self.step.generate_download_requests()) - finally: - shutil.rmtree(self.step.working_dir) - - def test_generate_download_reqs_perm_denied(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - - # make sure the permission denies OSError bubbles up - self.assertRaises(OSError, list, self.step.generate_download_requests()) - - def test_generate_download_reqs_ancestry_exists(self): - self.step.step_get_local_units.units_to_download.append({'image_id': 'image1'}) - self.step.working_dir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.step.working_dir, 'image1')) - # simulate the ancestry file already existing - open(os.path.join(self.step.working_dir, 'image1/ancestry'), 'w').close() - - try: - # there should only be 2 reqs instead of 3, since the ancestry file already exists - reqs = list(self.step.generate_download_requests()) - self.assertEqual(len(reqs), 2) - finally: - shutil.rmtree(self.step.working_dir) - - def test_sync(self): - with mock.patch.object(self.step, 'process_lifecycle') as mock_process: - report = self.step.sync() - - # make sure we called the process_lifecycle method - mock_process.assert_called_once_with() - # make sure it returned a report generated by the conduit - self.assertTrue(report is self.conduit.build_success_report.return_value) - - -class TestGerMetadataStep(unittest.TestCase): - def setUp(self): - super(TestGerMetadataStep, self).setUp() - self.working_dir = tempfile.mkdtemp() - self.repo = RepositoryModel('repo1') - self.repo.working_dir = self.working_dir - self.conduit = mock.MagicMock() - plugin_config = { - constants.CONFIG_KEY_UPSTREAM_NAME: 'pulp/crane', - importer_constants.KEY_FEED: 'http://pulpproject.org/', - } - self.config = PluginCallConfiguration({}, plugin_config) - - self.step = sync.GetMetadataStep(self.repo, self.conduit, self.config, self.working_dir) - self.step.parent = mock.MagicMock() - self.index = self.step.parent.index_repository - - def tearDown(self): - super(TestGerMetadataStep, self).tearDown() - shutil.rmtree(self.working_dir) - - def test_updates_tags(self): - self.index.get_tags.return_value = { - 'latest': 'abc1' - } - self.index.get_image_ids.return_value = ['abc123'] - self.step.parent.tags = {} - # make the ancestry file and put it in the expected place - os.makedirs(os.path.join(self.working_dir, 'abc123')) - with open(os.path.join(self.working_dir, 'abc123/ancestry'), 'w') as ancestry: - ancestry.write('["abc123"]') - - self.step.process_main() - - self.assertEqual(self.step.parent.tags, {'latest': 'abc123'}) - - def test_updates_available_units(self): - self.index.get_tags.return_value = { - 'latest': 'abc1' - } - self.index.get_image_ids.return_value = ['abc123'] - self.step.parent.tags = {} - # make the ancestry file and put it in the expected place - os.makedirs(os.path.join(self.working_dir, 'abc123')) - with open(os.path.join(self.working_dir, 'abc123/ancestry'), 'w') as ancestry: - ancestry.write('["abc123","xyz789"]') - - self.step.process_main() - - available_ids = [unit_key['image_id'] for unit_key in self.step.parent.available_units] - self.assertTrue('abc123' in available_ids) - self.assertTrue('xyz789' in available_ids) - - def test_expand_tags_no_abbreviations(self): - ids = ['abc123', 'xyz789'] - tags = {'foo': 'abc123', 'bar': 'abc123', 'baz': 'xyz789'} - - self.step.expand_tag_abbreviations(ids, tags) - self.assertEqual(tags['foo'], 'abc123') - self.assertEqual(tags['bar'], 'abc123') - self.assertEqual(tags['baz'], 'xyz789') - - def test_expand_tags_with_abbreviations(self): - ids = ['abc123', 'xyz789'] - tags = {'foo': 'abc', 'bar': 'abc123', 'baz': 'xyz'} - - self.step.expand_tag_abbreviations(ids, tags) - self.assertEqual(tags['foo'], 'abc123') - self.assertEqual(tags['bar'], 'abc123') - self.assertEqual(tags['baz'], 'xyz789') - - def test_find_and_read_ancestry_file(self): - # make the ancestry file and put it in the expected place - os.makedirs(os.path.join(self.working_dir, 'abc123')) - with open(os.path.join(self.working_dir, 'abc123/ancestry'), 'w') as ancestry: - ancestry.write('["abc123","xyz789"]') - - ancester_ids = self.step.find_and_read_ancestry_file('abc123', self.working_dir) - - self.assertEqual(ancester_ids, ['abc123', 'xyz789']) - - -class TestGetLocalImagesStep(unittest.TestCase): - - def setUp(self): - super(TestGetLocalImagesStep, self).setUp() - self.working_dir = tempfile.mkdtemp() - self.step = sync.GetLocalImagesStep(constants.IMPORTER_TYPE_ID, - constants.IMAGE_TYPE_ID, - ['image_id'], self.working_dir) - self.step.conduit = mock.MagicMock() - - def tearDown(self): - super(TestGetLocalImagesStep, self).tearDown() - shutil.rmtree(self.working_dir) - - def test_dict_to_unit(self): - unit = self.step._dict_to_unit({'image_id': 'abc123', 'parent_id': None, 'size': 12}) - - self.assertTrue(unit is self.step.conduit.init_unit.return_value) - self.step.conduit.init_unit.assert_called_once_with(constants.IMAGE_TYPE_ID, - {'image_id': 'abc123'}, - {'parent_id': None, 'size': 12}, - os.path.join(constants.IMAGE_TYPE_ID, - 'abc123')) - - -class TestSaveUnits(unittest.TestCase): - def setUp(self): - super(TestSaveUnits, self).setUp() - self.working_dir = tempfile.mkdtemp() - self.dest_dir = tempfile.mkdtemp() - self.step = sync.SaveUnits(self.working_dir) - self.step.repo = RepositoryModel('repo1') - self.step.conduit = mock.MagicMock() - self.step.parent = mock.MagicMock() - self.step.parent.step_get_local_units.units_to_download = [{'image_id': 'abc123'}] - - self.unit = Unit(constants.IMAGE_TYPE_ID, {'image_id': 'abc123'}, - {'parent': None, 'size': 2}, os.path.join(self.dest_dir, 'abc123')) - - def tearDown(self): - super(TestSaveUnits, self).tearDown() - shutil.rmtree(self.working_dir) - shutil.rmtree(self.dest_dir) - - def _write_empty_files(self): - os.makedirs(os.path.join(self.working_dir, 'abc123')) - open(os.path.join(self.working_dir, 'abc123/ancestry'), 'w').close() - open(os.path.join(self.working_dir, 'abc123/json'), 'w').close() - open(os.path.join(self.working_dir, 'abc123/layer'), 'w').close() - - def _write_files_legit_metadata(self): - os.makedirs(os.path.join(self.working_dir, 'abc123')) - open(os.path.join(self.working_dir, 'abc123/ancestry'), 'w').close() - open(os.path.join(self.working_dir, 'abc123/layer'), 'w').close() - # write just enough metadata to make the step happy - with open(os.path.join(self.working_dir, 'abc123/json'), 'w') as json_file: - json.dump({'Size': 2, 'Parent': 'xyz789'}, json_file) - - @mock.patch('pulp_docker.plugins.importers.tags.update_tags', spec_set=True) - def test_process_main_moves_files(self, mock_update_tags): - self._write_files_legit_metadata() - - with mock.patch.object(self.step, 'move_files') as mock_move_files: - self.step.process_main() - - expected_unit = self.step.conduit.init_unit.return_value - mock_move_files.assert_called_once_with(expected_unit) - - @mock.patch('pulp_docker.plugins.importers.tags.update_tags', spec_set=True) - def test_process_main_saves_unit(self, mock_update_tags): - self._write_files_legit_metadata() - - with mock.patch.object(self.step, 'move_files'): - self.step.process_main() - - expected_unit = self.step.conduit.init_unit.return_value - self.step.conduit.save_unit.assert_called_once_with(expected_unit) - - @mock.patch('pulp_docker.plugins.importers.tags.update_tags', spec_set=True) - def test_process_main_updates_tags(self, mock_update_tags): - self._write_files_legit_metadata() - self.step.parent.tags = {'latest': 'abc123'} - - with mock.patch.object(self.step, 'move_files'): - self.step.process_main() - - mock_update_tags.assert_called_once_with(self.step.repo.id, {'latest': 'abc123'}) - - def test_move_files_make_dir(self): - self._write_empty_files() - - self.step.move_files(self.unit) - - self.assertTrue(os.path.exists(os.path.join(self.dest_dir, 'abc123/ancestry'))) - self.assertTrue(os.path.exists(os.path.join(self.dest_dir, 'abc123/json'))) - self.assertTrue(os.path.exists(os.path.join(self.dest_dir, 'abc123/layer'))) - - self.assertFalse(os.path.exists(os.path.join(self.working_dir, 'abc123/ancestry'))) - self.assertFalse(os.path.exists(os.path.join(self.working_dir, 'abc123/json'))) - self.assertFalse(os.path.exists(os.path.join(self.working_dir, 'abc123/layer'))) - - def test_move_files_dir_exists(self): - self._write_empty_files() - os.makedirs(os.path.join(self.dest_dir, 'abc123')) - - self.step.move_files(self.unit) - - self.assertTrue(os.path.exists(os.path.join(self.dest_dir, 'abc123/ancestry'))) - self.assertTrue(os.path.exists(os.path.join(self.dest_dir, 'abc123/json'))) - self.assertTrue(os.path.exists(os.path.join(self.dest_dir, 'abc123/layer'))) - - self.assertFalse(os.path.exists(os.path.join(self.working_dir, 'abc123/ancestry'))) - self.assertFalse(os.path.exists(os.path.join(self.working_dir, 'abc123/json'))) - self.assertFalse(os.path.exists(os.path.join(self.working_dir, 'abc123/layer'))) - - def test_move_files_makedirs_fails(self): - self.unit.storage_path = '/a/b/c' - - # make sure that a permission denied error bubbles up - self.assertRaises(OSError, self.step.move_files, self.unit) diff --git a/plugins/test/unit/plugins/importers/test_upload.py b/plugins/test/unit/plugins/importers/test_upload.py deleted file mode 100644 index 52388327..00000000 --- a/plugins/test/unit/plugins/importers/test_upload.py +++ /dev/null @@ -1,165 +0,0 @@ -import json -import os -import shutil -import tempfile -import unittest - -import mock -from pulp.plugins.model import Unit -from pulp.server.managers import factory -from pulp.server.managers.repo.cud import RepoManager - -import data -from pulp_docker.common import constants -from pulp_docker.common.models import DockerImage -from pulp_docker.plugins.importers import upload - - -factory.initialize() - - -metadata = { - 'id1': {'parent': 'id2', 'size': 1024}, - 'id2': {'parent': 'id3', 'size': 1024}, - 'id3': {'parent': 'id4', 'size': 1024}, - 'id4': {'parent': None, 'size': 1024}, -} - -metadata_shared_parents_multiple_leaves = { - 'id1': {'parent': 'id2', 'size': 1024}, - 'id2': {'parent': 'id3', 'size': 1024}, - 'id3': {'parent': 'id5', 'size': 1024}, - 'id4': {'parent': 'id2', 'size': 1024}, - 'id5': {'parent': None, 'size': 1024}, -} - - -class TestGetModels(unittest.TestCase): - def test_full_metadata(self): - # Test for simple metadata - models = upload.get_models(metadata) - - self.assertEqual(len(models), len(metadata)) - for m in models: - self.assertTrue(isinstance(m, DockerImage)) - self.assertTrue(m.image_id in metadata) - - ids = [m.image_id for m in models] - self.assertEqual(set(ids), set(metadata.keys())) - - def test_full_metadata_shared_parents_multiple_leaves(self): - # Test for metadata having shared parents and multiple leaves - models = upload.get_models(metadata_shared_parents_multiple_leaves) - - self.assertEqual(len(models), len(metadata_shared_parents_multiple_leaves)) - for m in models: - self.assertTrue(isinstance(m, DockerImage)) - self.assertTrue(m.image_id in metadata_shared_parents_multiple_leaves) - - ids = [m.image_id for m in models] - self.assertEqual(set(ids), set(metadata_shared_parents_multiple_leaves.keys())) - - def test_mask(self): - # Test for simple metadata - models = upload.get_models(metadata, mask_id='id3') - - self.assertEqual(len(models), 2) - # make sure this only returns the first two and masks the others - for m in models: - self.assertTrue(m.image_id in ['id1', 'id2']) - - def test_mask_shared_parents_multiple_leaves(self): - # Test for metadata having shared parents and multiple leaves - models = upload.get_models(metadata_shared_parents_multiple_leaves, mask_id='id3') - - self.assertEqual(len(models), 3) - for m in models: - self.assertTrue(m.image_id in ['id1', 'id2', 'id4']) - - -class TestSaveModels(unittest.TestCase): - def setUp(self): - self.conduit = mock.MagicMock() - - @mock.patch('os.path.exists', return_value=True, spec_set=True) - def test_path_exists(self, mock_exists): - model = DockerImage('abc123', 'xyz789', 1024) - - upload.save_models(self.conduit, [model], (model.image_id,), data.busybox_tar_path) - - self.assertEqual(self.conduit.save_unit.call_count, 1) - self.conduit.init_unit.assert_called_once_with(constants.IMAGE_TYPE_ID, model.unit_key, - model.unit_metadata, model.relative_path) - - self.conduit.save_unit.assert_called_once_with(self.conduit.init_unit.return_value) - - def test_with_busybox(self): - models = [ - DockerImage(data.busybox_ids[0], data.busybox_ids[1], 1024), - ] - dest = tempfile.mkdtemp() - try: - # prepare some state - model_dest = os.path.join(dest, models[0].relative_path) - unit = Unit(DockerImage.TYPE_ID, models[0].unit_key, - models[0].unit_metadata, model_dest) - self.conduit.init_unit.return_value = unit - - # call the save, letting it write files to disk - upload.save_models(self.conduit, models, data.busybox_ids, data.busybox_tar_path) - - # assertions! - self.conduit.save_unit.assert_called_once_with(unit) - - # make sure the ancestry was computed and saved correctly - ancestry = json.load(open(os.path.join(model_dest, 'ancestry'))) - self.assertEqual(set(ancestry), set(data.busybox_ids)) - # make sure these files were moved into place - self.assertTrue(os.path.exists(os.path.join(model_dest, 'json'))) - self.assertTrue(os.path.exists(os.path.join(model_dest, 'layer'))) - finally: - shutil.rmtree(dest) - - -@mock.patch.object(RepoManager, 'get_repo_scratchpad', spec_set=True) -@mock.patch.object(RepoManager, 'update_repo_scratchpad', spec_set=True) -class TestUpdateTags(unittest.TestCase): - def test_basic_update(self, mock_update, mock_get): - mock_get.return_value = {'foo': 'other data that should not be part of the update'} - - upload.update_tags('repo1', data.busybox_tar_path) - - mock_update.assert_called_once_with('repo1', - {'tags': - [{constants.IMAGE_TAG_KEY: 'latest', - constants.IMAGE_ID_KEY: data.busybox_ids[0]}]}) - - def test_preserves_existing_tags(self, mock_update, mock_get): - mock_get.return_value = {'tags': [{constants.IMAGE_TAG_KEY: 'greatest', - constants.IMAGE_ID_KEY: data.busybox_ids[1]}]} - - upload.update_tags('repo1', data.busybox_tar_path) - - expected_tags = { - 'tags': [{constants.IMAGE_TAG_KEY: 'greatest', - constants.IMAGE_ID_KEY: data.busybox_ids[1]}, - {constants.IMAGE_TAG_KEY: 'latest', - constants.IMAGE_ID_KEY: data.busybox_ids[0]}] - } - mock_update.assert_called_once_with('repo1', expected_tags) - - def test_overwrite_existing_duplicate_tags(self, mock_update, mock_get): - mock_get.return_value = {'tags': [{constants.IMAGE_TAG_KEY: 'latest', - constants.IMAGE_ID_KEY: 'original_latest'}, - {constants.IMAGE_TAG_KEY: 'existing', - constants.IMAGE_ID_KEY: 'existing'}]} - - upload.update_tags('repo1', data.busybox_tar_path) - - expected_tags = { - 'tags': [{constants.IMAGE_TAG_KEY: 'existing', - constants.IMAGE_ID_KEY: 'existing'}, - {constants.IMAGE_TAG_KEY: 'latest', - constants.IMAGE_ID_KEY: data.busybox_ids[0]}] - } - mock_update.assert_called_once_with('repo1', expected_tags) diff --git a/plugins/test/unit/plugins/test_registry.py b/plugins/test/unit/plugins/test_registry.py deleted file mode 100644 index 90e12a54..00000000 --- a/plugins/test/unit/plugins/test_registry.py +++ /dev/null @@ -1,279 +0,0 @@ -from cStringIO import StringIO -import json -import os -import tempfile -import unittest - -import mock -from nectar.config import DownloaderConfig -from nectar.downloaders.threaded import HTTPThreadedDownloader -from nectar.report import DownloadReport -from nectar.request import DownloadRequest -import shutil - -from pulp_docker.plugins import registry - - -class TestInit(unittest.TestCase): - def test_init(self): - config = DownloaderConfig() - repo = registry.Repository('pulp/crane', config, 'http://pulpproject.org/', '/a/b/c') - - self.assertEqual(repo.name, 'pulp/crane') - self.assertEqual(repo.registry_url, 'http://pulpproject.org/') - self.assertEqual(repo.working_dir, '/a/b/c') - self.assertTrue(isinstance(repo.downloader, HTTPThreadedDownloader)) - - -class TestGetSinglePath(unittest.TestCase): - def setUp(self): - super(TestGetSinglePath, self).setUp() - self.config = DownloaderConfig() - self.repo = registry.Repository('pulp/crane', self.config, - 'http://pulpproject.org/', '/a/b/c') - - @mock.patch.object(HTTPThreadedDownloader, 'download_one') - def test_get_tags(self, mock_download_one): - body = json.dumps({'latest': 'abc123'}) - report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/tags', - StringIO(body)) - report.headers = {} - mock_download_one.return_value = report - - ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags') - - self.assertEqual(ret, {'latest': 'abc123'}) - self.assertEqual(mock_download_one.call_count, 1) - self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest)) - req = mock_download_one.call_args[0][0] - self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/tags') - - @mock.patch.object(HTTPThreadedDownloader, 'download_one') - def test_get_images(self, mock_download_one): - body = json.dumps(['abc123']) - report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images', - StringIO(body)) - report.headers = {} - mock_download_one.return_value = report - - ret = self.repo._get_single_path('/v1/repositories/pulp/crane/images') - - self.assertEqual(ret, ['abc123']) - self.assertEqual(mock_download_one.call_count, 1) - self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest)) - req = mock_download_one.call_args[0][0] - self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/images') - # make sure this header is set, which is required by the docker API in order - # to give us an auth token - self.assertEqual(req.headers[self.repo.DOCKER_TOKEN_HEADER], 'true') - - @mock.patch.object(HTTPThreadedDownloader, 'download_one') - def test_get_with_headers(self, mock_download_one): - body = json.dumps(['abc123']) - report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images', - StringIO(body)) - report.headers = { - self.repo.DOCKER_TOKEN_HEADER: 'token', - self.repo.DOCKER_ENDPOINT_HEADER: 'endpoint', - } - mock_download_one.return_value = report - - self.repo._get_single_path('/v1/repositories/pulp/crane/images') - - self.assertEqual(self.repo.token, 'token') - self.assertEqual(self.repo.endpoint, 'endpoint') - - @mock.patch.object(HTTPThreadedDownloader, 'download_one') - def test_get_single_path_failure(self, mock_download_one): - report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images', - StringIO('')) - report.headers = {} - report.state = report.DOWNLOAD_FAILED - mock_download_one.return_value = report - - self.assertRaises(IOError, self.repo._get_single_path, '/v1/repositories/pulp/crane/images') - - -class TestGetImageIDs(unittest.TestCase): - def setUp(self): - super(TestGetImageIDs, self).setUp() - self.config = DownloaderConfig() - self.repo = registry.Repository('pulp/crane', self.config, - 'http://pulpproject.org/', '/a/b/c') - - def test_returns_ids(self): - with mock.patch.object(self.repo, '_get_single_path') as mock_get: - mock_get.return_value = [{'id': 'abc123'}] - - ret = self.repo.get_image_ids() - - self.assertEqual(ret, ['abc123']) - mock_get.assert_called_once_with('/v1/repositories/pulp/crane/images') - - -class TestGetTags(unittest.TestCase): - def setUp(self): - super(TestGetTags, self).setUp() - self.config = DownloaderConfig() - self.repo = registry.Repository('pulp/crane', self.config, - 'http://pulpproject.org/', '/a/b/c') - - def test_returns_tags_as_dict(self): - with mock.patch.object(self.repo, '_get_single_path') as mock_get: - mock_get.return_value = {'latest': 'abc123'} - - ret = self.repo.get_tags() - - self.assertEqual(ret, {'latest': 'abc123'}) - mock_get.assert_called_once_with('/v1/repositories/pulp/crane/tags') - - def test_returns_tags_as_list(self): - with mock.patch.object(self.repo, '_get_single_path') as mock_get: - mock_get.return_value = [{'name': 'latest', 'layer': 'abc123'}] - - ret = self.repo.get_tags() - - self.assertEqual(ret, {'latest': 'abc123'}) - mock_get.assert_called_once_with('/v1/repositories/pulp/crane/tags') - - def test_adds_library_namespace(self): - self.repo.name = 'crane' - with mock.patch.object(self.repo, '_get_single_path') as mock_get: - mock_get.return_value = {'latest': 'abc123'} - - self.repo.get_tags() - - # make sure the "library" part of the path was added, which is a required - # quirk of the docker registry API - mock_get.assert_called_once_with('/v1/repositories/library/crane/tags') - - -class TestGetAncestry(unittest.TestCase): - def setUp(self): - super(TestGetAncestry, self).setUp() - self.working_dir = tempfile.mkdtemp() - self.config = DownloaderConfig() - self.repo = registry.Repository('pulp/crane', self.config, - 'http://pulpproject.org/', self.working_dir) - - def tearDown(self): - super(TestGetAncestry, self).tearDown() - shutil.rmtree(self.working_dir) - - def test_with_no_images(self): - with mock.patch.object(self.repo.downloader, 'download') as mock_download: - self.repo.get_ancestry([]) - - mock_download.assert_called_once_with([]) - - def test_makes_destination_dir(self): - with mock.patch.object(self.repo.downloader, 'download'): - self.repo.get_ancestry(['abc123']) - - self.assertTrue(os.path.isdir(os.path.join(self.working_dir, 'abc123'))) - - def test_dir_already_exists(self): - with mock.patch.object(self.repo.downloader, 'download'): - self.repo.get_ancestry(['abc123']) - - self.assertTrue(os.path.isdir(os.path.join(self.working_dir, 'abc123'))) - - def test_error_making_dir(self): - self.repo.working_dir = '/a/b/c' - # this should be a permission denied error, which should be allowed to bubble up - self.assertRaises(OSError, self.repo.get_ancestry, ['abc123']) - - def test_makes_request(self): - with mock.patch.object(self.repo.downloader, 'download') as mock_download: - self.repo.get_ancestry(['abc123']) - - self.assertEqual(mock_download.call_count, 1) - self.assertEqual(len(mock_download.call_args[0][0]), 1) - req = mock_download.call_args[0][0][0] - self.assertEqual(req.url, 'http://pulpproject.org/v1/images/abc123/ancestry') - self.assertEqual(req.destination, os.path.join(self.working_dir, 'abc123/ancestry')) - - def test_adds_auth_header(self): - self.repo.token = 'letmein' - with mock.patch.object(self.repo.downloader, 'download') as mock_download: - self.repo.get_ancestry(['abc123']) - - self.assertEqual(mock_download.call_count, 1) - self.assertEqual(len(mock_download.call_args[0][0]), 1) - req = mock_download.call_args[0][0][0] - self.assertEqual(req.headers['Authorization'], 'Token letmein') - - def test_uses_endpoint(self): - self.repo.endpoint = 'redhat.com' - with mock.patch.object(self.repo.downloader, 'download') as mock_download: - self.repo.get_ancestry(['abc123']) - - self.assertEqual(mock_download.call_count, 1) - self.assertEqual(len(mock_download.call_args[0][0]), 1) - req = mock_download.call_args[0][0][0] - self.assertEqual(req.url, 'http://redhat.com/v1/images/abc123/ancestry') - - def test_failed_request(self): - self.repo.listener.failed_reports.append( - DownloadReport('http://redhat.com/v1/images/abc123/ancestry', '/a/b/c')) - with mock.patch.object(self.repo.downloader, 'download'): - self.assertRaises(IOError, self.repo.get_ancestry, ['abc123']) - - -class TestAddAuthHeader(unittest.TestCase): - def setUp(self): - super(TestAddAuthHeader, self).setUp() - self.config = DownloaderConfig() - self.repo = registry.Repository('pulp/crane', self.config, - 'http://pulpproject.org/', '/a/b/') - self.request = DownloadRequest('http://pulpproject.org', '/a/b/c') - - def test_add_token(self): - self.repo.token = 'letmein' - - self.repo.add_auth_header(self.request) - - self.assertEqual(self.request.headers['Authorization'], 'Token letmein') - - def test_without_token(self): - self.request.headers = {} - self.repo.add_auth_header(self.request) - - self.assertTrue('Authorization' not in self.request.headers) - - def test_does_not_clobber_other_headers(self): - self.repo.token = 'letmein' - self.request.headers = {'foo': 'bar'} - - self.repo.add_auth_header(self.request) - - self.assertEqual(self.request.headers['Authorization'], 'Token letmein') - self.assertEqual(self.request.headers['foo'], 'bar') - - -class TestGetImageURL(unittest.TestCase): - def setUp(self): - super(TestGetImageURL, self).setUp() - self.config = DownloaderConfig() - self.repo = registry.Repository('pulp/crane', self.config, - 'http://pulpproject.org/', '/a/b/') - - def test_without_endpoint(self): - url = self.repo.get_image_url() - - self.assertEqual(url, self.repo.registry_url) - - def test_with_endpoint(self): - self.repo.endpoint = 'redhat.com' - - url = self.repo.get_image_url() - - self.assertEqual(url, 'http://redhat.com/') - - def test_preserves_https(self): - self.repo.registry_url = 'https://pulpproject.org/' - self.repo.endpoint = 'redhat.com' - - url = self.repo.get_image_url() - - self.assertEqual(url, 'https://redhat.com/') diff --git a/plugins/types/docker.json b/plugins/types/docker.json deleted file mode 100644 index c747d32f..00000000 --- a/plugins/types/docker.json +++ /dev/null @@ -1,9 +0,0 @@ -{"types": [ - { - "id": "docker_image", - "display_name": "Docker Image", - "description": "Docker Image", - "unit_key": ["image_id"], - "search_indexes": [] - } -]} diff --git a/pulp-dev.py b/pulp-dev.py deleted file mode 100755 index 2f037b1a..00000000 --- a/pulp-dev.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import optparse -import os -import sys - -WARNING_COLOR = '\033[31m' -WARNING_RESET = '\033[0m' - -DIRS = ('/var/lib/pulp/published/docker/web',) - -# -# Str entry assumes same src and dst relative path. -# Tuple entry is explicit (src, dst) -# -# Please keep alphabetized and by subproject - -# Standard directories -DIR_PLUGINS = '/usr/lib/pulp/plugins' - -LINKS = ( - ('plugins/etc/httpd/conf.d/pulp_docker.conf', '/etc/httpd/conf.d/pulp_docker.conf'), - ('plugins/etc/pulp/server/plugins.conf.d/docker_distributor.json', - '/etc/pulp/server/plugins.conf.d/docker_distributor.json'), - ('plugins/types/docker.json', DIR_PLUGINS + '/types/docker.json'), -) - - -def parse_cmdline(): - """ - Parse and validate the command line options. - """ - parser = optparse.OptionParser() - - parser.add_option('-I', '--install', - action='store_true', - help='install pulp development files') - parser.add_option('-U', '--uninstall', - action='store_true', - help='uninstall pulp development files') - parser.add_option('-D', '--debug', - action='store_true', - help=optparse.SUPPRESS_HELP) - - parser.set_defaults(install=False, - uninstall=False, - debug=True) - - opts, args = parser.parse_args() - - if opts.install and opts.uninstall: - parser.error('both install and uninstall specified') - - if not (opts.install or opts.uninstall): - parser.error('neither install or uninstall specified') - - return (opts, args) - - -def warning(msg): - print "%s%s%s" % (WARNING_COLOR, msg, WARNING_RESET) - - -def debug(opts, msg): - if not opts.debug: - return - sys.stderr.write('%s\n' % msg) - - -def create_dirs(opts): - for d in DIRS: - if os.path.exists(d) and os.path.isdir(d): - debug(opts, 'skipping %s exists' % d) - continue - debug(opts, 'creating directory: %s' % d) - os.makedirs(d, 0777) - - -def getlinks(): - links = [] - for l in LINKS: - if isinstance(l, (list, tuple)): - src = l[0] - dst = l[1] - else: - src = l - dst = os.path.join('/', l) - links.append((src, dst)) - return links - - -def install(opts): - warnings = [] - create_dirs(opts) - # Ensure the directory is owned by apache - os.system('chown -R apache:apache /var/lib/pulp/published/docker') - - currdir = os.path.abspath(os.path.dirname(__file__)) - for src, dst in getlinks(): - warning_msg = create_link(opts, os.path.join(currdir, src), dst) - if warning_msg: - warnings.append(warning_msg) - - if warnings: - print "\n***\nPossible problems: Please read below\n***" - for w in warnings: - warning(w) - return os.EX_OK - - -def uninstall(opts): - for src, dst in getlinks(): - debug(opts, 'removing link: %s' % dst) - if not os.path.islink(dst): - debug(opts, '%s does not exist, skipping' % dst) - continue - os.unlink(dst) - - return os.EX_OK - - -def create_link(opts, src, dst): - if not os.path.lexists(dst): - return _create_link(opts, src, dst) - - if not os.path.islink(dst): - return "[%s] is not a symbolic link as we expected, please adjust if this " \ - "is not what you intended." % (dst) - - if not os.path.exists(os.readlink(dst)): - warning('BROKEN LINK: [%s] attempting to delete and fix it to point to %s.' % (dst, src)) - try: - os.unlink(dst) - return _create_link(opts, src, dst) - except: - msg = "[%s] was a broken symlink, failed to delete and relink to [%s], " \ - "please fix this manually"\ - % (dst, src) - return msg - - debug(opts, 'verifying link: %s points to %s' % (dst, src)) - dst_stat = os.stat(dst) - src_stat = os.stat(src) - if dst_stat.st_ino != src_stat.st_ino: - msg = "[%s] is pointing to [%s] which is different than the intended target [%s]"\ - % (dst, os.readlink(dst), src) - return msg - - -def _create_link(opts, src, dst): - debug(opts, 'creating link: %s pointing to %s' % (dst, src)) - try: - os.symlink(src, dst) - except OSError, e: - msg = "Unable to create symlink for [%s] pointing to [%s], received error: <%s>"\ - % (dst, src, e) - return msg - -# ----------------------------------------------------------------------------- - -if __name__ == '__main__': - # TODO add something to check for permissions - opts, args = parse_cmdline() - if opts.install: - sys.exit(install(opts)) - if opts.uninstall: - sys.exit(uninstall(opts)) diff --git a/pulp-docker.spec b/pulp-docker.spec deleted file mode 100644 index abe69280..00000000 --- a/pulp-docker.spec +++ /dev/null @@ -1,162 +0,0 @@ -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} -%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} - -Name: pulp-docker -Version: 0.2.1 -Release: 0.2.beta%{?dist} -Summary: Support for Docker layers in the Pulp platform -Group: Development/Languages -License: GPLv2 -URL: http://pulpproject.org -Source0: https://fedorahosted.org/releases/p/u/%{name}/%{name}-%{version}.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildArch: noarch -BuildRequires: python2-devel -BuildRequires: python-setuptools -BuildRequires: rpm-python - -%description -Provides a collection of platform plugins and admin client extensions to -provide docker support - -%prep -%setup -q - -%build -pushd common -%{__python} setup.py build -popd - -pushd extensions_admin -%{__python} setup.py build -popd - -pushd plugins -%{__python} setup.py build -popd - -%install -rm -rf %{buildroot} - -mkdir -p %{buildroot}/%{_sysconfdir}/pulp/ - -pushd common -%{__python} setup.py install --skip-build --root %{buildroot} -popd - -pushd extensions_admin -%{__python} setup.py install --skip-build --root %{buildroot} -popd - -pushd plugins -%{__python} setup.py install --skip-build --root %{buildroot} -popd - -mkdir -p %{buildroot}/%{_usr}/lib/pulp/plugins/types -mkdir -p %{buildroot}/%{_var}/lib/pulp/published/docker/app/ -mkdir -p %{buildroot}/%{_var}/lib/pulp/published/docker/export/ -mkdir -p %{buildroot}/%{_var}/lib/pulp/published/docker/web/ - -cp -R plugins/etc/httpd %{buildroot}/%{_sysconfdir} -# Types -cp -R plugins/types/* %{buildroot}/%{_usr}/lib/pulp/plugins/types/ - -mkdir -p %{buildroot}/%{_bindir} - -# Remove tests -rm -rf %{buildroot}/%{python_sitelib}/test - -%clean -rm -rf %{buildroot} - - - -# ---- Docker Common ----------------------------------------------------------- - -%package -n python-pulp-docker-common -Summary: Pulp Docker support common library -Group: Development/Languages -Requires: python-pulp-common >= 2.4.0 -Requires: python-setuptools - -%description -n python-pulp-docker-common -Common libraries for python-pulp-docker - -%files -n python-pulp-docker-common -%defattr(-,root,root,-) -%dir %{python_sitelib}/pulp_docker -%{python_sitelib}/pulp_docker/__init__.py* -%{python_sitelib}/pulp_docker/common/ -%dir %{python_sitelib}/pulp_docker/extensions -%{python_sitelib}/pulp_docker/extensions/__init__.py* -%{python_sitelib}/pulp_docker_common*.egg-info -%doc COPYRIGHT LICENSE AUTHORS - - -# ---- Plugins ----------------------------------------------------------------- -%package plugins -Summary: Pulp Docker plugins -Group: Development/Languages -Requires: python-pulp-common >= 2.4.0 -Requires: python-pulp-docker-common = %{version} -Requires: pulp-server >= 2.4.0 -Requires: python-setuptools - -%description plugins -Provides a collection of platform plugins that extend the Pulp platform -to provide Docker specific support - -%files plugins - -%defattr(-,root,root,-) -%{python_sitelib}/pulp_docker/plugins/ -%config(noreplace) %{_sysconfdir}/httpd/conf.d/pulp_docker.conf -%{_usr}/lib/pulp/plugins/types/docker.json -%{python_sitelib}/pulp_docker_plugins*.egg-info - -%defattr(-,apache,apache,-) -%{_var}/lib/pulp/published/docker/ - -%doc COPYRIGHT LICENSE AUTHORS - - -# ---- Admin Extensions -------------------------------------------------------- -%package admin-extensions -Summary: The Pulp Docker admin client extensions -Group: Development/Languages -Requires: python-pulp-common >= 2.4.0 -Requires: python-pulp-docker-common = %{version} -Requires: pulp-admin-client >= 2.4.0 -Requires: python-setuptools - -%description admin-extensions -pulp-admin extensions for docker support - -%files admin-extensions -%defattr(-,root,root,-) -%{python_sitelib}/pulp_docker/extensions/admin/ -%{python_sitelib}/pulp_docker_extensions_admin*.egg-info -%doc COPYRIGHT LICENSE AUTHORS - - -%changelog -* Thu Oct 02 2014 Chris Duryee 0.2.1-0.2.beta -- making the default size None when a layer's metadata lacks the Size attribute - (mhrivnak@redhat.com) -- adding several publish directories that need to be in the package - (mhrivnak@redhat.com) - -* Thu Sep 11 2014 Chris Duryee 0.2.1-0.1.alpha -- declare correct package version in spec file (cduryee@redhat.com) - -* Tue Sep 09 2014 Chris Duryee 0.2.0-1 - bump to 0.2.0 -- - -* Mon Sep 08 2014 Chris Duryee 0.1.2-1 -- adding cancellation support (mhrivnak@redhat.com) -- adding sync (mhrivnak@redhat.com) - -* Mon Jul 07 2014 Chris Duryee 0.1.1-1 -- new package built with tito - diff --git a/pulp_docker/__init__.py b/pulp_docker/__init__.py new file mode 100644 index 00000000..c6978041 --- /dev/null +++ b/pulp_docker/__init__.py @@ -0,0 +1,3 @@ +__version__ = '4.0.0b8.dev' + +default_app_config = 'pulp_docker.app.PulpDockerPluginAppConfig' diff --git a/pulp_docker/app/__init__.py b/pulp_docker/app/__init__.py new file mode 100644 index 00000000..59731091 --- /dev/null +++ b/pulp_docker/app/__init__.py @@ -0,0 +1,8 @@ +from pulpcore.plugin import PulpPluginAppConfig + + +class PulpDockerPluginAppConfig(PulpPluginAppConfig): + """Entry point for the docker plugin.""" + + name = 'pulp_docker.app' + label = 'docker' diff --git a/pulp_docker/app/authorization.py b/pulp_docker/app/authorization.py new file mode 100644 index 00000000..3f583a99 --- /dev/null +++ b/pulp_docker/app/authorization.py @@ -0,0 +1,178 @@ +import base64 +import hashlib +import random +import uuid + +import jwt + +from datetime import datetime +from aiohttp import web + +from django.conf import settings +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization + +from pulpcore.plugin.content import Handler + +TOKEN_EXPIRATION_TIME = 300 + +KNOWN_SERVICES = [settings.CONTENT_ORIGIN] +ANONYMOUS_USER = '' +EMPTY_ACCESS_SCOPE = '::' + + +class AuthorizationService(Handler): + """ + A class responsible for generating and managing a Bearer token. + + This class represents a token server which manages and grants permissions + according to a user's scope. + """ + + async def generate_token(self, request): + """ + Generate a Bearer token. + + A signed JSON web token is generated in this method. The structure of the token is + adjusted according the documentation https://docs.docker.com/registry/spec/auth/jwt/. + + Args: + request(:class:`~aiohttp.web.Request`): The request to prepare a response for. + + Returns: + class:`aiohttp.web_response.Response`: A newly generated Bearer token. + + """ + with open(settings.PUBLIC_KEY_PATH, 'rb') as public_key: + kid = self.generate_kid_header(public_key.read()) + + current_datetime = datetime.now() + + token_queries = TokenRequestQueries.init_from(request) + access = self.determine_access(ANONYMOUS_USER, token_queries.scope) + token_server = getattr(settings, 'TOKEN_SERVER', '') + claim_set = self._generate_claim_set( + access=[access], + audience=token_queries.service, + issued_at=int(current_datetime.timestamp()), + issuer=token_server, + subject=ANONYMOUS_USER + ) + + with open(settings.PRIVATE_KEY_PATH, 'rb') as private_key: + binary_token = jwt.encode( + claim_set, private_key.read(), + algorithm=settings.TOKEN_SIGNATURE_ALGORITHM, + headers={'kid': kid} + ) + token = binary_token.decode('utf8') + current_datetime_utc = current_datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + return web.json_response({ + 'expires_in': TOKEN_EXPIRATION_TIME, + 'issued_at': current_datetime_utc, + 'token': token + }) + + def generate_kid_header(self, public_key): + """Generate kid header in a libtrust compatible format.""" + decoded_key = self._convert_key_format_from_pem_to_der(public_key) + truncated_sha256 = hashlib.sha256(decoded_key).hexdigest()[:30].encode('utf8') + encoded_base32 = base64.b32encode(truncated_sha256).decode('utf8') + return self._split_into_encoded_groups(encoded_base32) + + def _convert_key_format_from_pem_to_der(self, public_key): + key_in_pem_format = serialization.load_pem_public_key(public_key, default_backend()) + key_in_der_format = key_in_pem_format.public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.SubjectPublicKeyInfo + ) + return key_in_der_format + + def _split_into_encoded_groups(self, encoded_base32): + """Split encoded and truncated base32 into 12 groups separated by ':'.""" + kid = encoded_base32[:4] + for index, char in enumerate(encoded_base32[4:], start=0): + if index % 4 == 0: + kid += ':' + char + else: + kid += char + return kid + + def determine_access(self, user, scope): + """ + Determine access permissions for a corresponding user. + + This method determines whether the user has a valid access permission or not. + The determination is based on role based access control. For now, the access + is given out to anybody because the role based access control is not implemented + yet. + + Args: + user (str): A name of the user who is trying to access a registry. + scope (str): A requested scope. + + Returns: + list: An intersected set of the requested and the allowed access. + + """ + typ, name, actions = scope.split(':') + actions_list = actions.split(',') + permitted_actions = list(set(actions_list).intersection(['pull'])) + return {'type': typ, 'name': name, 'actions': permitted_actions} + + def _generate_claim_set(self, issuer, issued_at, subject, audience, access): + token_id = str(uuid.UUID(int=random.getrandbits(128), version=4)) + expiration = issued_at + TOKEN_EXPIRATION_TIME + return { + 'access': access, + 'aud': audience, + 'exp': expiration, + 'iat': issued_at, + 'iss': issuer, + 'jti': token_id, + 'nbf': issued_at, + 'sub': subject + } + + +class TokenRequestQueries: + """A data class that holds data retrieved from the request queries.""" + + def __init__(self, scope, service): + """ + Store a scope and a service. + + Args: + scope (str): A requested scope. + service (str): A service that request a Bearer token. + + """ + self.scope = scope + self.service = service + + @classmethod + def init_from(cls, request): + """ + Initialize the actual class with data retrieved from the request queries. + + In this method, a validity and a presence of required queries (scope, service) + is checked as well. If the scope is not specified, the method checks if a user + is trying to access root endpoint only. Then, the scope is not relevant anymore + and initialized to empty type, name, and requested actions ('::'). + """ + try: + scope = request.query['scope'] + except KeyError: + if request.match_info: + raise web.HTTPBadRequest(reason='A scope was not provided.') + else: + scope = EMPTY_ACCESS_SCOPE + + try: + service = request.query['service'] + except KeyError: + raise web.HTTPBadRequest(reason='A service name was not provided.') + if service not in KNOWN_SERVICES: + raise web.HTTPBadRequest(reason='A provided service is unknown.') + + return cls(scope, service) diff --git a/pulp_docker/app/content.py b/pulp_docker/app/content.py new file mode 100644 index 00000000..f93debfa --- /dev/null +++ b/pulp_docker/app/content.py @@ -0,0 +1,17 @@ +from aiohttp import web + +from pulpcore.content import app +from pulp_docker.app.registry import Registry +from pulp_docker.app.authorization import AuthorizationService + +registry = Registry() + +app.add_routes([web.get('/v2/', Registry.serve_v2)]) +app.add_routes([web.get(r'/v2/{path:.+}/blobs/sha256:{digest:.+}', registry.get_by_digest)]) +app.add_routes([web.get(r'/v2/{path:.+}/manifests/sha256:{digest:.+}', registry.get_by_digest)]) +app.add_routes([web.get(r'/v2/{path:.+}/manifests/{tag_name}', registry.get_tag)]) +app.add_routes([web.get(r'/v2/{path:.+}/tags/list', registry.tags_list)]) + +authorization_service = AuthorizationService() + +app.add_routes([web.get('/token', authorization_service.generate_token)]) diff --git a/pulp_docker/app/docker_convert.py b/pulp_docker/app/docker_convert.py new file mode 100644 index 00000000..e4868d93 --- /dev/null +++ b/pulp_docker/app/docker_convert.py @@ -0,0 +1,281 @@ +import os +import base64 +import binascii +import datetime +import ecdsa +import hashlib +import itertools +import json +import logging +from collections import namedtuple +from jwkest import jws, jwk, ecc + +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings + +from pulp_docker.constants import MEDIA_TYPE + +log = logging.getLogger(__name__) + +FS_Layer = namedtuple("FS_Layer", "layer_id uncompressed_digest history") + + +class Schema1ConverterWrapper: + """An abstraction around creating new manifests of the format schema 1.""" + + def __init__(self, tag, accepted_media_types, path): + """Store a tag object, accepted media type, and path.""" + self.path = path + self.tag = tag + self.accepted_media_types = accepted_media_types + self.name = path + + def convert(self): + """Convert a manifest to schema 1.""" + if self.tag.tagged_manifest.media_type == MEDIA_TYPE.MANIFEST_V2: + converted_schema = self._convert_schema(self.tag.tagged_manifest) + return converted_schema, True, self.tag.tagged_manifest.digest + elif self.tag.tagged_manifest.media_type == MEDIA_TYPE.MANIFEST_LIST: + legacy = self._get_legacy_manifest() + if legacy.media_type in self.accepted_media_types: + # return legacy without conversion + return legacy, False, legacy.digest + elif legacy.media_type == MEDIA_TYPE.MANIFEST_V2: + converted_schema = self._convert_schema(legacy) + return converted_schema, True, legacy.digest + else: + raise RuntimeError() + + def _convert_schema(self, manifest): + config_dict = _get_config_dict(manifest) + manifest_dict = _get_manifest_dict(manifest) + + converter = ConverterS2toS1( + manifest_dict, + config_dict, + name=self.name, + tag=self.tag.name + ) + return converter.convert() + + def _get_legacy_manifest(self): + ml = self.tag.tagged_manifest.listed_manifests.all() + for manifest in ml: + m = manifest.manifest_lists.first() + if m.architecture == 'amd64' and m.os == 'linux': + return m.manifest_list + + raise RuntimeError() + + +class ConverterS2toS1: + """ + Converter class from schema 2 to schema 1. + + Initialize it with a manifest and a config layer JSON documents, + and call convert() to obtain the signed manifest, as a JSON-encoded string. + """ + + EMPTY_LAYER = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + + def __init__(self, manifest, config_layer, name, tag): + """ + Initializer needs a manifest and a config layer as JSON documents. + """ + self.name = name + self.tag = tag + self.manifest = manifest + self.config_layer = config_layer + self.fs_layers = [] + self.history = [] + + def convert(self): + """ + Convert manifest from schema 2 to schema 1 + """ + self.compute_layers() + manifest = dict( + name=self.name, tag=self.tag, architecture=self.config_layer['architecture'], + schemaVersion=1, fsLayers=self.fs_layers, history=self.history + ) + key = jwk.ECKey().load_key(ecc.P256) + key.kid = getKeyId(key) + manifData = sign(manifest, key) + return manifData + + def compute_layers(self): + """ + Compute layers to be present in the converted image. + Empty (throwaway) layers will be created to store image metadata + """ + # Layers in v2s1 are in reverse order from v2s2 + fs_layers = self._compute_fs_layers() + self.fs_layers = [dict(blobSum=x[0]) for x in fs_layers] + # Compute v1 compatibility + parent = None + history_entries = self.history = [] + + fs_layers_count = len(fs_layers) + # Reverse list so we can compute parent/child properly + fs_layers.reverse() + for i, fs_layer in enumerate(fs_layers): + layer_id = self._compute_layer_id(fs_layer.layer_id, fs_layer.uncompressed_digest, i) + config = self._compute_v1_compatibility_config( + layer_id, fs_layer, last_layer=(i == fs_layers_count - 1)) + if parent is not None: + config['parent'] = parent + parent = layer_id + history_entries.append(dict(v1Compatibility=_jsonDumpsCompact(config))) + # Reverse again for proper order + history_entries.reverse() + + def _compute_fs_layers(self): + """Utility function to return a list of FS_Layer objects""" + layers = reversed(self.manifest['layers']) + config_layer_history = reversed(self.config_layer['history']) + diff_ids = reversed(self.config_layer['rootfs']['diff_ids']) + fs_layers = [] + curr_compressed_dig = next(layers)['digest'] + curr_uncompressed_dig = next(diff_ids) + for curr_hist in config_layer_history: + if curr_hist.get("empty_layer"): + layer_id = self.EMPTY_LAYER + uncompressed_dig = None + else: + layer_id = curr_compressed_dig + uncompressed_dig = curr_uncompressed_dig + try: + curr_compressed_dig = next(layers)['digest'] + curr_uncompressed_dig = next(diff_ids) + except StopIteration: + curr_compressed_dig = self.EMPTY_LAYER + curr_uncompressed_dig = None + fs_layers.append(FS_Layer(layer_id, uncompressed_dig, curr_hist)) + return fs_layers + + def _compute_v1_compatibility_config(self, layer_id, fs_layer, last_layer=False): + """Utility function to compute the v1 compatibility""" + if last_layer: + # The whole config layer becomes part of the v1compatibility + # (minus history and rootfs) + config = dict(self.config_layer) + config.pop("history", None) + config.pop("rootfs", None) + else: + config = dict(created=fs_layer.history['created'], + container_config=dict(Cmd=fs_layer.history['created_by'])) + if fs_layer.uncompressed_digest is None: + config['throwaway'] = True + config['id'] = layer_id + return config + + @classmethod + def _compute_layer_id(cls, compressed_dig, uncompressed_dig, layer_index): + """ + We need to make up an image ID for each layer. + We will digest: + * the compressed digest of the layer + * the uncompressed digest (if present; it will be missing for throw-away layers) + * the zero-padded integer of the layer number + The last one is added so we can get different image IDs for throw-away layers. + """ + dig = hashlib.sha256(compressed_dig.encode("ascii")) + if uncompressed_dig: + dig.update(uncompressed_dig.encode("ascii")) + layer_count = "%06d" % layer_index + dig.update(layer_count.encode("ascii")) + layer_id = dig.hexdigest() + return layer_id + + +def _jsonDumps(data): + return json.dumps(data, indent=3, sort_keys=True, separators=(',', ': ')) + + +def _jsonDumpsCompact(data): + return json.dumps(data, sort_keys=True, separators=(',', ':')) + + +def sign(data, key): + """ + Sign the JSON document with a elliptic curve key + """ + jdata = _jsonDumps(data) + now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + header = dict(alg="ES256", jwk=key.serialize()) + protected = dict(formatLength=len(jdata) - 2, + formatTail=jws.b64encode_item(jdata[-2:]), + time=now) + _jws = jws.JWS(jdata, **header) + protectedHeader, payload, signature = _jws.sign_compact([key], protected=protected).split(".") + signatures = [dict(header=header, signature=signature, protected=protectedHeader)] + jsig = _jsonDumps(dict(signatures=signatures))[1:-2] + arr = [jdata[:-2], ',', jsig, jdata[-2:]] + # Add the signature block at the end of the json string, keeping the + # formatting + jdata2 = ''.join(arr) + return jdata2 + + +def getKeyId(key): + """ + DER-encode the key and represent it in the format XXXX:YYYY:... + """ + derRepr = toDer(key) + shaRepr = hashlib.sha256(derRepr).digest()[:30] + b32Repr = base64.b32encode(shaRepr).decode() + return ':'.join(byN(b32Repr, 4)) + + +def toDer(key): + """Return the DER-encoded representation of the key""" + point = b"\x00\x04" + number2string(key.x, key.curve.bytes) + \ + number2string(key.y, key.curve.bytes) + der = ecdsa.der + curveEncodedOid = der.encode_oid(1, 2, 840, 10045, 3, 1, 7) + return der.encode_sequence( + der.encode_sequence(ecdsa.keys.encoded_oid_ecPublicKey, curveEncodedOid), + der.encode_bitstring(point)) + + +def byN(strobj, N): + """ + Yield consecutive substrings of length N from string strobj + """ + it = iter(strobj) + while True: + substr = ''.join(itertools.islice(it, N)) + if not substr: + return + yield substr + + +def number2string(num, order): + """ + Hex-encode the number and return a zero-padded (to the left) to a total + length of 2*order + """ + # convert to hex + nhex = "%x" % num + # Zero-pad to the left so the length of the resulting unhexified string is order + nhex = nhex.rjust(2 * order, '0') + return binascii.unhexlify(nhex) + + +def _get_config_dict(manifest): + try: + config_artifact = manifest.config_blob._artifacts.get() + except ObjectDoesNotExist: + raise RuntimeError() + return _get_dict(config_artifact) + + +def _get_manifest_dict(manifest): + manifest_artifact = manifest._artifacts.get() + return _get_dict(manifest_artifact) + + +def _get_dict(artifact): + with open(os.path.join(settings.MEDIA_ROOT, artifact.file.path)) as json_file: + json_string = json_file.read() + return json.loads(json_string) diff --git a/pulp_docker/app/downloaders.py b/pulp_docker/app/downloaders.py new file mode 100644 index 00000000..e0792825 --- /dev/null +++ b/pulp_docker/app/downloaders.py @@ -0,0 +1,151 @@ +from gettext import gettext as _ +from logging import getLogger +from urllib import parse +import aiohttp +import asyncio +import backoff +import json +import re + +from aiohttp.client_exceptions import ClientResponseError + +from pulpcore.plugin.download import http_giveup, HttpDownloader + + +log = getLogger(__name__) + + +class RegistryAuthHttpDownloader(HttpDownloader): + """ + Custom Downloader that automatically handles Token Based and Basic Authentication. + + Additionally, use custom headers from DeclarativeArtifact.extra_data['headers'] + """ + + registry_auth = {'bearer': None, 'basic': None} + token_lock = asyncio.Lock() + + def __init__(self, *args, **kwargs): + """ + Initialize the downloader. + """ + self.remote = kwargs.pop('remote') + super().__init__(*args, **kwargs) + + @backoff.on_exception(backoff.expo, ClientResponseError, max_tries=10, giveup=http_giveup) + async def _run(self, handle_401=True, extra_data=None): + """ + Download, validate, and compute digests on the `url`. This is a coroutine. + + This method is decorated with a backoff-and-retry behavior to retry HTTP 429 errors. It + retries with exponential backoff 10 times before allowing a final exception to be raised. + + This method provides the same return object type and documented in + :meth:`~pulpcore.plugin.download.BaseDownloader._run`. + + Args: + handle_401(bool): If true, catch 401, request a new token and retry. + + """ + headers = {} + repo_name = None + if extra_data is not None: + headers = extra_data.get('headers', headers) + repo_name = extra_data.get('repo_name', None) + this_token = self.registry_auth['bearer'] + basic_auth = self.registry_auth['basic'] + auth_headers = self.auth_header(this_token, basic_auth) + headers.update(auth_headers) + # aiohttps does not allow to send auth argument and auth header together + self.session._default_auth = None + async with self.session.get(self.url, headers=headers, proxy=self.proxy) as response: + try: + response.raise_for_status() + except ClientResponseError as e: + response_auth_header = response.headers.get('www-authenticate') + # Need to retry request + if handle_401 and e.status == 401 and response_auth_header is not None: + # check if bearer or basic + if 'Bearer' in response_auth_header: + # Token has not been updated during request + if self.registry_auth['bearer'] is None or \ + self.registry_auth['bearer'] == this_token: + + self.registry_auth['bearer'] = None + await self.update_token(response_auth_header, this_token, repo_name) + return await self._run(handle_401=False) + elif 'Basic' in response_auth_header: + if self.remote.username: + basic = aiohttp.BasicAuth(self.remote.username, self.remote.password) + self.registry_auth['basic'] = basic.encode() + return await self._run(handle_401=False) + else: + raise + to_return = await self._handle_response(response) + await response.release() + self.response_headers = response.headers + + if self._close_session_on_finalize: + self.session.close() + return to_return + + async def update_token(self, response_auth_header, used_token, repo_name): + """ + Update the Bearer token to be used with all requests. + """ + async with self.token_lock: + if self.registry_auth['bearer'] is not None and \ + self.registry_auth['bearer'] == used_token: + return + log.info("Updating bearer token") + bearer_info_string = response_auth_header[len("Bearer "):] + bearer_info_list = re.split(',(?=[^=,]+=)', bearer_info_string) + + # The remaining string consists of comma seperated key=value pairs + auth_query_dict = {} + for key, value in (item.split('=') for item in bearer_info_list): + # The value is a string within a string, ex: '"value"' + auth_query_dict[key] = json.loads(value) + try: + token_base_url = auth_query_dict.pop('realm') + except KeyError: + raise IOError(_("No realm specified for token auth challenge.")) + + # self defense strategy in cases when registry does not provide the scope + if 'scope' not in auth_query_dict: + auth_query_dict['scope'] = 'repository:{0}:pull'.format(repo_name) + + # Construct a url with query parameters containing token auth challenge info + parsed_url = parse.urlparse(token_base_url) + # Add auth query params to query dict and urlencode into a string + new_query = parse.urlencode({**parse.parse_qs(parsed_url.query), **auth_query_dict}) + updated_parsed = parsed_url._replace(query=new_query) + token_url = parse.urlunparse(updated_parsed) + headers = {} + if self.remote.username: + # for private repos + basic = aiohttp.BasicAuth(self.remote.username, self.remote.password).encode() + headers['Authorization'] = basic + async with self.session.get(token_url, headers=headers, proxy=self.proxy, + raise_for_status=True) as token_response: + token_data = await token_response.text() + + self.registry_auth['bearer'] = json.loads(token_data)['token'] + + @staticmethod + def auth_header(token, basic_auth): + """ + Create an auth header that optionally includes a bearer token or basic auth. + + Args: + auth (str): Bearer token or Basic auth to use in header + + Returns: + dictionary: containing Authorization headers or {} if Authorizationis is None. + + """ + if token is not None: + return {'Authorization': 'Bearer {token}'.format(token=token)} + elif basic_auth is not None: + return {'Authorization': basic_auth} + return {} diff --git a/pulp_docker/app/migrations/0001_initial.py b/pulp_docker/app/migrations/0001_initial.py new file mode 100644 index 00000000..80cd6f73 --- /dev/null +++ b/pulp_docker/app/migrations/0001_initial.py @@ -0,0 +1,123 @@ +# Generated by Django 2.2.4 on 2019-08-12 16:09 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('core', '0003_remove_upload_completed'), + ] + + operations = [ + migrations.CreateModel( + name='Blob', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.Content')), + ('digest', models.CharField(max_length=255)), + ('media_type', models.CharField(choices=[('application/vnd.docker.container.image.v1+json', 'application/vnd.docker.container.image.v1+json'), ('application/vnd.docker.image.rootfs.diff.tar.gzip', 'application/vnd.docker.image.rootfs.diff.tar.gzip'), ('application/vnd.docker.image.rootfs.foreign.diff.tar.gzip', 'application/vnd.docker.image.rootfs.foreign.diff.tar.gzip')], max_length=80)), + ], + options={ + 'unique_together': {('digest',)}, + }, + bases=('core.content',), + ), + migrations.CreateModel( + name='BlobManifest', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.CreateModel( + name='DockerRemote', + fields=[ + ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.Remote')), + ('upstream_name', models.CharField(db_index=True, max_length=255)), + ('include_foreign_layers', models.BooleanField(default=False)), + ('whitelist_tags', models.TextField(null=True)), + ], + options={ + 'abstract': False, + }, + bases=('core.remote',), + ), + migrations.CreateModel( + name='Manifest', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.Content')), + ('digest', models.CharField(max_length=255)), + ('schema_version', models.IntegerField()), + ('media_type', models.CharField(choices=[('application/vnd.docker.distribution.manifest.v1+json', 'application/vnd.docker.distribution.manifest.v1+json'), ('application/vnd.docker.distribution.manifest.v2+json', 'application/vnd.docker.distribution.manifest.v2+json'), ('application/vnd.docker.distribution.manifest.list.v2+json', 'application/vnd.docker.distribution.manifest.list.v2+json')], max_length=60)), + ('blobs', models.ManyToManyField(through='docker.BlobManifest', to='docker.Blob')), + ('config_blob', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='config_blob', to='docker.Blob')), + ], + bases=('core.content',), + ), + migrations.CreateModel( + name='ManifestListManifest', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('architecture', models.CharField(max_length=255)), + ('os', models.CharField(max_length=255)), + ('os_version', models.CharField(blank=True, default='', max_length=255)), + ('os_features', models.TextField(blank=True, default='')), + ('features', models.TextField(blank=True, default='')), + ('variant', models.CharField(blank=True, default='', max_length=255)), + ('image_manifest', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='image_manifests', to='docker.Manifest')), + ('manifest_list', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='manifest_lists', to='docker.Manifest')), + ], + options={ + 'unique_together': {('image_manifest', 'manifest_list')}, + }, + ), + migrations.AddField( + model_name='manifest', + name='listed_manifests', + field=models.ManyToManyField(through='docker.ManifestListManifest', to='docker.Manifest'), + ), + migrations.CreateModel( + name='DockerDistribution', + fields=[ + ('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.BaseDistribution')), + ('repository', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Repository')), + ('repository_version', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.RepositoryVersion')), + ], + options={ + 'abstract': False, + }, + bases=('core.basedistribution',), + ), + migrations.AddField( + model_name='blobmanifest', + name='manifest', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='blob_manifests', to='docker.Manifest'), + ), + migrations.AddField( + model_name='blobmanifest', + name='manifest_blob', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='manifest_blobs', to='docker.Blob'), + ), + migrations.CreateModel( + name='Tag', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.Content')), + ('name', models.CharField(db_index=True, max_length=255)), + ('tagged_manifest', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tagged_manifests', to='docker.Manifest')), + ], + options={ + 'unique_together': {('name', 'tagged_manifest')}, + }, + bases=('core.content',), + ), + migrations.AlterUniqueTogether( + name='manifest', + unique_together={('digest',)}, + ), + migrations.AlterUniqueTogether( + name='blobmanifest', + unique_together={('manifest', 'manifest_blob')}, + ), + ] diff --git a/pulp_docker/app/migrations/0002_docker_related_names.py b/pulp_docker/app/migrations/0002_docker_related_names.py new file mode 100644 index 00000000..551a464b --- /dev/null +++ b/pulp_docker/app/migrations/0002_docker_related_names.py @@ -0,0 +1,79 @@ +# Generated by Django 2.2.4 on 2019-08-19 20:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('docker', '0001_initial'), + ] + + operations = [ + migrations.AlterModelOptions( + name='blob', + options={'default_related_name': '%(app_label)s_%(model_name)s'}, + ), + migrations.AlterModelOptions( + name='dockerdistribution', + options={'default_related_name': '%(app_label)s_%(model_name)s'}, + ), + migrations.AlterModelOptions( + name='dockerremote', + options={'default_related_name': '%(app_label)s_%(model_name)s'}, + ), + migrations.AlterModelOptions( + name='manifest', + options={'default_related_name': '%(app_label)s_%(model_name)s'}, + ), + migrations.AlterModelOptions( + name='tag', + options={'default_related_name': '%(app_label)s_%(model_name)s'}, + ), + migrations.AlterField( + model_name='blob', + name='content_ptr', + field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='docker_blob', serialize=False, to='core.Content'), + ), + migrations.AlterField( + model_name='dockerdistribution', + name='basedistribution_ptr', + field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='docker_dockerdistribution', serialize=False, to='core.BaseDistribution'), + ), + migrations.AlterField( + model_name='dockerdistribution', + name='repository', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='docker_dockerdistribution', to='core.Repository'), + ), + migrations.AlterField( + model_name='dockerdistribution', + name='repository_version', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='docker_dockerdistribution', to='core.RepositoryVersion'), + ), + migrations.AlterField( + model_name='dockerremote', + name='remote_ptr', + field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='docker_dockerremote', serialize=False, to='core.Remote'), + ), + migrations.AlterField( + model_name='manifest', + name='blobs', + field=models.ManyToManyField(related_name='docker_manifest', through='docker.BlobManifest', to='docker.Blob'), + ), + migrations.AlterField( + model_name='manifest', + name='content_ptr', + field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='docker_manifest', serialize=False, to='core.Content'), + ), + migrations.AlterField( + model_name='manifest', + name='listed_manifests', + field=models.ManyToManyField(related_name='docker_manifest', through='docker.ManifestListManifest', to='docker.Manifest'), + ), + migrations.AlterField( + model_name='tag', + name='content_ptr', + field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='docker_tag', serialize=False, to='core.Content'), + ), + ] diff --git a/pulp_docker/app/migrations/0003_index_content.py b/pulp_docker/app/migrations/0003_index_content.py new file mode 100644 index 00000000..09f01d6d --- /dev/null +++ b/pulp_docker/app/migrations/0003_index_content.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2019-10-02 15:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('docker', '0002_docker_related_names'), + ] + + operations = [ + migrations.AlterField( + model_name='blob', + name='digest', + field=models.CharField(db_index=True, max_length=255), + ), + migrations.AlterField( + model_name='manifest', + name='digest', + field=models.CharField(db_index=True, max_length=255), + ), + ] diff --git a/pulp_docker/app/migrations/0004_whitelist_tags_list.py b/pulp_docker/app/migrations/0004_whitelist_tags_list.py new file mode 100644 index 00000000..105788aa --- /dev/null +++ b/pulp_docker/app/migrations/0004_whitelist_tags_list.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2019-10-22 08:03 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('docker', '0003_index_content'), + ] + + operations = [ + migrations.AlterField( + model_name='dockerremote', + name='whitelist_tags', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255, null=True), null=True, size=None), + ), + ] diff --git a/common/test/unit/__init__.py b/pulp_docker/app/migrations/__init__.py similarity index 100% rename from common/test/unit/__init__.py rename to pulp_docker/app/migrations/__init__.py diff --git a/pulp_docker/app/models.py b/pulp_docker/app/models.py new file mode 100644 index 00000000..cb7b12ac --- /dev/null +++ b/pulp_docker/app/models.py @@ -0,0 +1,280 @@ +import re + +from logging import getLogger + +from django.db import models +from django.contrib.postgres import fields + +from pulpcore.plugin.download import DownloaderFactory +from pulpcore.plugin.models import Content, Remote, RepositoryVersion, RepositoryVersionDistribution + +from . import downloaders +from pulp_docker.constants import MEDIA_TYPE + + +logger = getLogger(__name__) + + +class Blob(Content): + """ + A blob defined within a manifest. + + The actual blob file is stored as an artifact. + + Fields: + digest (models.CharField): The blob digest. + media_type (models.CharField): The blob media type. + + Relations: + manifest (models.ForeignKey): Many-to-one relationship with Manifest. + """ + + TYPE = 'blob' + + BLOB_CHOICES = ( + (MEDIA_TYPE.CONFIG_BLOB, MEDIA_TYPE.CONFIG_BLOB), + (MEDIA_TYPE.REGULAR_BLOB, MEDIA_TYPE.REGULAR_BLOB), + (MEDIA_TYPE.FOREIGN_BLOB, MEDIA_TYPE.FOREIGN_BLOB), + ) + digest = models.CharField(max_length=255, db_index=True) + media_type = models.CharField( + max_length=80, + choices=BLOB_CHOICES + ) + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + unique_together = ('digest',) + + +class Manifest(Content): + """ + A docker manifest. + + This content has one artifact. + + Fields: + digest (models.CharField): The manifest digest. + schema_version (models.IntegerField): The docker schema version. + media_type (models.CharField): The manifest media type. + + Relations: + blobs (models.ManyToManyField): Many-to-many relationship with Blob. + config_blob (models.ForeignKey): Blob that contains configuration for this Manifest. + listed_manifests (models.ManyToManyField): Many-to-many relationship with Manifest. This + field is used only for a manifest-list type Manifests. + """ + + TYPE = 'manifest' + + MANIFEST_CHOICES = ( + (MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V1), + (MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_V2), + (MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.MANIFEST_LIST), + ) + digest = models.CharField(max_length=255, db_index=True) + schema_version = models.IntegerField() + media_type = models.CharField( + max_length=60, + choices=MANIFEST_CHOICES) + + blobs = models.ManyToManyField(Blob, through='BlobManifest') + config_blob = models.ForeignKey(Blob, related_name='config_blob', + null=True, on_delete=models.CASCADE) + + # Order matters for through fields, (source, target) + listed_manifests = models.ManyToManyField( + "self", + through='ManifestListManifest', + symmetrical=False, + through_fields=('image_manifest', 'manifest_list') + ) + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + unique_together = ('digest',) + + +class BlobManifest(models.Model): + """ + Many-to-many relationship between Blobs and Manifests. + """ + + manifest = models.ForeignKey( + Manifest, related_name='blob_manifests', on_delete=models.CASCADE) + manifest_blob = models.ForeignKey( + Blob, related_name='manifest_blobs', on_delete=models.CASCADE) + + class Meta: + unique_together = ('manifest', 'manifest_blob') + + +class ManifestListManifest(models.Model): + """ + The manifest referenced by a manifest list. + + Fields: + architecture (models.CharField): The platform architecture. + variant (models.CharField): The platform variant. + features (models.TextField): The platform features. + os (models.CharField): The platform OS name. + os_version (models.CharField): The platform OS version. + os_features (models.TextField): The platform OS features. + + Relations: + manifest (models.ForeignKey): Many-to-one relationship with Manifest. + manifest_list (models.ForeignKey): Many-to-one relationship with ManifestList. + """ + + architecture = models.CharField(max_length=255) + os = models.CharField(max_length=255) + os_version = models.CharField(max_length=255, default='', blank=True) + os_features = models.TextField(default='', blank=True) + features = models.TextField(default='', blank=True) + variant = models.CharField(max_length=255, default='', blank=True) + + image_manifest = models.ForeignKey( + Manifest, related_name='image_manifests', on_delete=models.CASCADE) + manifest_list = models.ForeignKey( + Manifest, related_name='manifest_lists', on_delete=models.CASCADE) + + class Meta: + unique_together = ('image_manifest', 'manifest_list') + + +class Tag(Content): + """ + A tagged Manifest. + + Fields: + name (models.CharField): The tag name. + + Relations: + tagged_manifest (models.ForeignKey): A referenced Manifest. + + """ + + TYPE = 'tag' + + name = models.CharField(max_length=255, db_index=True) + + tagged_manifest = models.ForeignKey( + Manifest, null=True, related_name='tagged_manifests', on_delete=models.CASCADE) + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + unique_together = ( + ('name', 'tagged_manifest'), + ) + + +class DockerRemote(Remote): + """ + A Remote for DockerContent. + + Fields: + upstream_name (models.CharField): The name of the image at the remote. + include_foreign_layers (models.BooleanField): Foreign layers in the remote + are included. They are not included by default. + """ + + upstream_name = models.CharField(max_length=255, db_index=True) + include_foreign_layers = models.BooleanField(default=False) + whitelist_tags = fields.ArrayField( + models.CharField(max_length=255, null=True), + null=True + ) + + TYPE = 'docker' + + @property + def download_factory(self): + """ + Return the DownloaderFactory which can be used to generate asyncio capable downloaders. + + Upon first access, the DownloaderFactory is instantiated and saved internally. + + Plugin writers are expected to override when additional configuration of the + DownloaderFactory is needed. + + Returns: + DownloadFactory: The instantiated DownloaderFactory to be used by + get_downloader() + + """ + try: + return self._download_factory + except AttributeError: + self._download_factory = DownloaderFactory( + self, + downloader_overrides={ + 'http': downloaders.RegistryAuthHttpDownloader, + 'https': downloaders.RegistryAuthHttpDownloader, + } + ) + return self._download_factory + + def get_downloader(self, remote_artifact=None, url=None, **kwargs): + """ + Get a downloader from either a RemoteArtifact or URL that is configured with this Remote. + + This method accepts either `remote_artifact` or `url` but not both. At least one is + required. If neither or both are passed a ValueError is raised. + + Args: + remote_artifact (:class:`~pulpcore.app.models.RemoteArtifact`): The RemoteArtifact to + download. + url (str): The URL to download. + kwargs (dict): This accepts the parameters of + :class:`~pulpcore.plugin.download.BaseDownloader`. + + Raises: + ValueError: If neither remote_artifact and url are passed, or if both are passed. + + Returns: + subclass of :class:`~pulpcore.plugin.download.BaseDownloader`: A downloader that + is configured with the remote settings. + + """ + kwargs['remote'] = self + return super().get_downloader(remote_artifact=remote_artifact, url=url, **kwargs) + + @property + def namespaced_upstream_name(self): + """ + Returns an upstream Docker repository name with a namespace. + + For upstream repositories that do not have a namespace, the convention is to use 'library' + as the namespace. + """ + # Docker's registry aligns non-namespaced images to the library namespace. + docker_registry = re.search(r'registry[-,\w]*.docker.io', self.url, re.IGNORECASE) + if '/' not in self.upstream_name and docker_registry: + return 'library/{name}'.format(name=self.upstream_name) + else: + return self.upstream_name + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + + +class DockerDistribution(RepositoryVersionDistribution): + """ + A docker distribution defines how a publication is distributed by Pulp's webserver. + """ + + TYPE = 'docker' + + def get_repository_version(self): + """ + Returns the repository version that is supposed to be served by this DockerDistribution. + """ + if self.repository: + return RepositoryVersion.latest(self.repository) + elif self.repository_version: + return self.repository_version + else: + return None + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" diff --git a/pulp_docker/app/registry.py b/pulp_docker/app/registry.py new file mode 100644 index 00000000..32dff918 --- /dev/null +++ b/pulp_docker/app/registry.py @@ -0,0 +1,256 @@ +import logging +import os + +from aiohttp import web +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from multidict import MultiDict + +from pulpcore.plugin.content import Handler, PathNotResolved +from pulpcore.plugin.models import ContentArtifact +from pulp_docker.app.models import DockerDistribution, Tag +from pulp_docker.app.docker_convert import Schema1ConverterWrapper +from pulp_docker.app.token_verification import TokenVerifier +from pulp_docker.constants import MEDIA_TYPE + +log = logging.getLogger(__name__) + +v2_headers = MultiDict() +v2_headers['Docker-Distribution-API-Version'] = 'registry/2.0' + + +class ArtifactNotFound(Exception): + """ + The artifact associated with a published-artifact does not exist. + """ + + pass + + +class Registry(Handler): + """ + A set of handlers for the Docker v2 API. + """ + + distribution_model = DockerDistribution + + @staticmethod + async def get_accepted_media_types(request): + """ + Returns a list of media types from the Accept headers. + + Args: + request(:class:`~aiohttp.web.Request`): The request to extract headers from. + + Returns: + List of media types supported by the client. + + """ + accepted_media_types = [] + for header, values in request.raw_headers: + if header == b'Accept': + values = [v.strip().decode('UTF-8') for v in values.split(b",")] + accepted_media_types.extend(values) + return accepted_media_types + + @staticmethod + def _base_paths(path): + """ + Get a list of base paths used to match a distribution. + + Args: + path (str): The path component of the URL. + + Returns: + list: Of base paths. + + """ + return [path] + + @staticmethod + async def _dispatch(path, headers): + """ + Stream a file back to the client. + + Stream the bits. + + Args: + path (str): The fully qualified path to the file to be served. + headers (dict): + + Returns: + StreamingHttpResponse: Stream the requested content. + + """ + full_headers = MultiDict() + + full_headers['Content-Type'] = headers['Content-Type'] + full_headers['Docker-Content-Digest'] = headers['Docker-Content-Digest'] + full_headers['Docker-Distribution-API-Version'] = 'registry/2.0' + full_headers['Content-Length'] = os.path.getsize(path) + full_headers['Content-Disposition'] = 'attachment; filename={n}'.format( + n=os.path.basename(path)) + file_response = web.FileResponse(path, headers=full_headers) + return file_response + + @staticmethod + async def serve_v2(request): + """ + Handler for Docker Registry v2 root. + """ + Registry.verify_token(request, 'pull') + + return web.json_response({}, headers=v2_headers) + + async def tags_list(self, request): + """ + Handler for Docker Registry v2 tags/list API. + """ + Registry.verify_token(request, 'pull') + + path = request.match_info['path'] + distribution = self._match_distribution(path) + tags = {'name': path, 'tags': set()} + repository_version = distribution.get_repository_version() + for c in repository_version.content: + c = c.cast() + if isinstance(c, Tag): + tags['tags'].add(c.name) + tags['tags'] = list(tags['tags']) + return web.json_response(tags, headers=v2_headers) + + async def get_tag(self, request): + """ + Match the path and stream either Manifest or ManifestList. + + Args: + request(:class:`~aiohttp.web.Request`): The request to prepare a response for. + + Raises: + PathNotResolved: The path could not be matched to a published file. + PermissionError: When not permitted. + + Returns: + :class:`aiohttp.web.StreamResponse` or :class:`aiohttp.web.FileResponse`: The response + streamed back to the client. + + """ + Registry.verify_token(request, 'pull') + + path = request.match_info['path'] + tag_name = request.match_info['tag_name'] + distribution = self._match_distribution(path) + repository_version = distribution.get_repository_version() + accepted_media_types = await Registry.get_accepted_media_types(request) + + try: + tag = Tag.objects.get( + pk__in=repository_version.content, + name=tag_name, + ) + except ObjectDoesNotExist: + raise PathNotResolved(tag_name) + + if tag.tagged_manifest.media_type == MEDIA_TYPE.MANIFEST_V1: + return_media_type = MEDIA_TYPE.MANIFEST_V1_SIGNED + response_headers = {'Content-Type': return_media_type, + 'Docker-Content-Digest': tag.tagged_manifest.digest} + return await Registry.dispatch_tag(tag, response_headers) + + if tag.tagged_manifest.media_type in accepted_media_types: + return_media_type = tag.tagged_manifest.media_type + response_headers = {'Content-Type': return_media_type, + 'Docker-Content-Digest': tag.tagged_manifest.digest} + return await Registry.dispatch_tag(tag, response_headers) + + return await Registry.dispatch_converted_schema(tag, accepted_media_types, path) + + @staticmethod + async def dispatch_tag(tag, response_headers): + """ + Finds an artifact associated with a Tag and sends it to the client. + + Args: + tag: Tag + response_headers (dict): dictionary that contains the 'Content-Type' header to send + with the response + + Returns: + :class:`aiohttp.web.StreamResponse` or :class:`aiohttp.web.FileResponse`: The response + streamed back to the client. + + """ + try: + artifact = tag._artifacts.get() + except ObjectDoesNotExist: + raise ArtifactNotFound(tag.name) + else: + return await Registry._dispatch(os.path.join(settings.MEDIA_ROOT, artifact.file.name), + response_headers) + + @staticmethod + async def dispatch_converted_schema(tag, accepted_media_types, path): + """ + Convert a manifest from the format schema 2 to the format schema 1. + + The format is converted on-the-go and created resources are not stored for further uses. + The conversion is made after each request which does not accept the format for schema 2. + + Args: + tag: A tag object which contains reference to tagged manifests and config blobs. + accepted_media_types: Accepted media types declared in the accept header. + path: A path of a repository. + + Raises: + PathNotResolved: There was not found a valid conversion for the specified tag. + + Returns: + :class:`aiohttp.web.StreamResponse` or :class:`aiohttp.web.Response`: The response + streamed back to the client. + + """ + schema1_converter = Schema1ConverterWrapper(tag, accepted_media_types, path) + try: + schema, converted, digest = schema1_converter.convert() + except RuntimeError: + raise PathNotResolved(tag.name) + response_headers = {'Docker-Content-Digest': digest, + 'Content-Type': MEDIA_TYPE.MANIFEST_V1_SIGNED, + 'Docker-Distribution-API-Version': 'registry/2.0'} + if not converted: + return await Registry.dispatch_tag(schema, response_headers) + + return web.Response(text=schema, headers=response_headers) + + async def get_by_digest(self, request): + """ + Return a response to the "GET" action. + """ + Registry.verify_token(request, 'pull') + + path = request.match_info['path'] + digest = "sha256:{digest}".format(digest=request.match_info['digest']) + distribution = self._match_distribution(path) + repository_version = distribution.get_repository_version() + log.info(digest) + try: + ca = ContentArtifact.objects.get(content__in=repository_version.content, + relative_path=digest) + headers = {'Content-Type': ca.content.cast().media_type, + 'Docker-Content-Digest': ca.content.cast().digest} + except ObjectDoesNotExist: + raise PathNotResolved(path) + else: + artifact = ca.artifact + if artifact: + return await Registry._dispatch(os.path.join(settings.MEDIA_ROOT, + artifact.file.name), + headers) + else: + return await self._stream_content_artifact(request, web.StreamResponse(), ca) + + @staticmethod + def verify_token(request, access_action): + """Verify a Bearer token.""" + token_verifier = TokenVerifier(request, access_action) + token_verifier.verify() diff --git a/pulp_docker/app/serializers.py b/pulp_docker/app/serializers.py new file mode 100644 index 00000000..d3983545 --- /dev/null +++ b/pulp_docker/app/serializers.py @@ -0,0 +1,373 @@ +from gettext import gettext as _ + +from django.conf import settings + +from rest_framework import serializers + +from pulpcore.plugin.models import ( + Remote, + Repository, + RepositoryVersion, +) +from pulpcore.plugin.serializers import ( + DetailRelatedField, + NestedRelatedField, + RemoteSerializer, + RepositoryVersionDistributionSerializer, + SingleArtifactContentSerializer, + RelatedField, + validate_unknown_fields, +) + +from . import models + + +class TagSerializer(SingleArtifactContentSerializer): + """ + Serializer for Tags. + """ + + name = serializers.CharField(help_text="Tag name") + tagged_manifest = DetailRelatedField( + many=False, + help_text="Manifest that is tagged", + view_name='docker-manifests-detail', + queryset=models.Manifest.objects.all() + ) + + class Meta: + fields = SingleArtifactContentSerializer.Meta.fields + ( + 'name', + 'tagged_manifest', + ) + model = models.Tag + + +class ManifestSerializer(SingleArtifactContentSerializer): + """ + Serializer for Manifests. + """ + + digest = serializers.CharField(help_text="sha256 of the Manifest file") + schema_version = serializers.IntegerField(help_text="Docker schema version") + media_type = serializers.CharField(help_text="Docker media type of the file") + listed_manifests = DetailRelatedField( + many=True, + help_text="Manifests that are referenced by this Manifest List", + view_name='docker-manifests-detail', + queryset=models.Manifest.objects.all() + ) + blobs = DetailRelatedField( + many=True, + help_text="Blobs that are referenced by this Manifest", + view_name='docker-blobs-detail', + queryset=models.Blob.objects.all() + ) + config_blob = DetailRelatedField( + many=False, + help_text="Blob that contains configuration for this Manifest", + view_name='docker-blobs-detail', + queryset=models.Blob.objects.all() + ) + + class Meta: + fields = SingleArtifactContentSerializer.Meta.fields + ( + 'digest', + 'schema_version', + 'media_type', + 'listed_manifests', + 'config_blob', + 'blobs', + ) + model = models.Manifest + + +class BlobSerializer(SingleArtifactContentSerializer): + """ + Serializer for Blobs. + """ + + digest = serializers.CharField(help_text="sha256 of the Blob file") + media_type = serializers.CharField(help_text="Docker media type of the file") + + class Meta: + fields = SingleArtifactContentSerializer.Meta.fields + ( + 'digest', + 'media_type', + ) + model = models.Blob + + +class RegistryPathField(serializers.CharField): + """ + Serializer Field for the registry_path field of the DockerDistribution. + """ + + def to_representation(self, value): + """ + Converts a base_path into a registry path. + """ + host = settings.CONTENT_ORIGIN + return ''.join([host, '/', value]) + + +class DockerRemoteSerializer(RemoteSerializer): + """ + A Serializer for DockerRemote. + """ + + upstream_name = serializers.CharField( + required=True, + allow_blank=False, + help_text=_("Name of the upstream repository") + ) + whitelist_tags = serializers.ListField( + child=serializers.CharField(max_length=255), + allow_null=True, + required=False, + help_text=_("A list of whitelisted tags to sync") + ) + + policy = serializers.ChoiceField( + help_text=""" + immediate - All manifests and blobs are downloaded and saved during a sync. + on_demand - Only tags and manifests are downloaded. Blobs are not + downloaded until they are requested for the first time by a client. + streamed - Blobs are streamed to the client with every request and never saved. + """, + choices=Remote.POLICY_CHOICES, + default=Remote.IMMEDIATE + ) + + class Meta: + fields = RemoteSerializer.Meta.fields + ('upstream_name', 'whitelist_tags',) + model = models.DockerRemote + + +class DockerDistributionSerializer(RepositoryVersionDistributionSerializer): + """ + A serializer for DockerDistribution. + """ + + registry_path = RegistryPathField( + source='base_path', read_only=True, + help_text=_('The Registry hostame:port/name/ to use with docker pull command defined by ' + 'this distribution.') + ) + + class Meta: + model = models.DockerDistribution + fields = tuple(set(RepositoryVersionDistributionSerializer.Meta.fields) - {'base_url'}) + ( + 'registry_path',) + + +class TagOperationSerializer(serializers.Serializer): + """ + A base serializer for tagging and untagging manifests. + """ + + repository = RelatedField( + required=True, + view_name='repositories-detail', + queryset=Repository.objects.all(), + help_text='A URI of the repository.' + ) + tag = serializers.CharField( + required=True, + help_text='A tag name' + ) + + def validate(self, data): + """ + Validate data passed through a request call. + + Check if a repository has got a reference to a latest repository version. A + new dictionary object is initialized by the passed data and altered by a latest + repository version. + """ + new_data = {} + new_data.update(data) + + latest_version = RepositoryVersion.latest(data['repository']) + if not latest_version: + raise serializers.ValidationError( + _("The latest repository version of '{}' was not found" + .format(data['repository'])) + ) + + new_data['latest_version'] = latest_version + return new_data + + +class TagImageSerializer(TagOperationSerializer): + """ + A serializer for parsing and validating data associated with a manifest tagging. + """ + + digest = serializers.CharField( + required=True, + help_text='sha256 of the Manifest file' + ) + + def validate(self, data): + """ + Validate data passed through a request call. + + Manifest with a corresponding digest is retrieved from a database and stored + in the dictionary to avoid querying the database in the ViewSet again. The + method checks if the tag exists within the repository. + """ + new_data = super().validate(data) + + try: + manifest = models.Manifest.objects.get( + pk__in=new_data['latest_version'].content.all(), + digest=new_data['digest'] + ) + except models.Manifest.DoesNotExist: + raise serializers.ValidationError( + _("A manifest with the digest '{}' does not " + "exist in the latest repository version '{}'" + .format(new_data['digest'], new_data['latest_version'])) + ) + + new_data['manifest'] = manifest + return new_data + + +class UnTagImageSerializer(TagOperationSerializer): + """ + A serializer for parsing and validating data associated with a manifest untagging. + """ + + def validate(self, data): + """ + Validate data passed through a request call. + + The method checks if the tag exists within the latest repository version. + """ + new_data = super().validate(data) + + try: + models.Tag.objects.get( + pk__in=new_data['latest_version'].content.all(), + name=new_data['tag'] + ) + except models.Tag.DoesNotExist: + raise serializers.ValidationError( + _("The tag '{}' does not exist in the latest repository version '{}'" + .format(new_data['tag'], new_data['latest_version'])) + ) + + return new_data + + +class RecursiveManageSerializer(serializers.Serializer): + """ + Serializer for adding and removing content to/from a Docker repository. + """ + + repository = serializers.HyperlinkedRelatedField( + required=True, + help_text=_('A URI of the repository to add content.'), + queryset=Repository.objects.all(), + view_name='repositories-detail', + label=_('Repository'), + error_messages={ + 'required': _('The repository URI must be specified.') + } + ) + content_units = serializers.ListField( + help_text=_('A list of content units to operate on.'), + write_only=True, + required=False + ) + + +class CopySerializer(serializers.Serializer): + """ + Serializer for copying units from a source repository to a destination repository. + """ + + source_repository = serializers.HyperlinkedRelatedField( + help_text=_('A URI of the repository to copy content from.'), + queryset=Repository.objects.all(), + view_name='repositories-detail', + label=_('Repository'), + write_only=True, + required=False, + ) + source_repository_version = NestedRelatedField( + help_text=_('A URI of the repository version to copy content from.'), + view_name='versions-detail', + lookup_field='number', + parent_lookup_kwargs={'repository_pk': 'repository__pk'}, + queryset=models.RepositoryVersion.objects.all(), + write_only=True, + required=False, + ) + destination_repository = serializers.HyperlinkedRelatedField( + required=True, + help_text=_('A URI of the repository to copy content to.'), + queryset=Repository.objects.all(), + view_name='repositories-detail', + label=_('Repository'), + error_messages={ + 'required': _('Destination repository URI must be specified.') + } + ) + + def validate(self, data): + """Ensure that source_repository or source_rpository_version is pass, but not both.""" + if hasattr(self, 'initial_data'): + validate_unknown_fields(self.initial_data, self.fields) + + repository = data.pop('source_repository', None) + repository_version = data.get('source_repository_version') + if not repository and not repository_version: + raise serializers.ValidationError( + _("Either the 'repository' or 'repository_version' need to be specified")) + elif not repository and repository_version: + return data + elif repository and not repository_version: + version = models.RepositoryVersion.latest(repository) + if version: + new_data = {'source_repository_version': version} + new_data.update(data) + return new_data + else: + raise serializers.ValidationError( + detail=_('Source repository has no version available to copy content from')) + raise serializers.ValidationError( + _("Either the 'repository' or 'repository_version' need to be specified " + "but not both.") + ) + + +class TagCopySerializer(CopySerializer): + """ + Serializer for copying tags from a source repository to a destination repository. + """ + + names = serializers.ListField( + required=False, + allow_null=False, + help_text="A list of tag names to copy." + ) + + +class ManifestCopySerializer(CopySerializer): + """ + Serializer for copying manifests from a source repository to a destination repository. + """ + + digests = serializers.ListField( + required=False, + allow_null=False, + help_text="A list of manifest digests to copy." + ) + media_types = serializers.MultipleChoiceField( + choices=models.Manifest.MANIFEST_CHOICES, + required=False, + help_text="A list of media_types to copy." + ) diff --git a/pulp_docker/app/tasks/__init__.py b/pulp_docker/app/tasks/__init__.py new file mode 100644 index 00000000..8d9dacb8 --- /dev/null +++ b/pulp_docker/app/tasks/__init__.py @@ -0,0 +1,6 @@ +from .distribution import create, delete, update # noqa +from .recursive_add import recursive_add_content # noqa +from .recursive_remove import recursive_remove_content # noqa +from .synchronize import synchronize # noqa +from .tag import tag_image # noqa +from .untag import untag_image # noqa diff --git a/pulp_docker/app/tasks/distribution.py b/pulp_docker/app/tasks/distribution.py new file mode 100644 index 00000000..4cb5516b --- /dev/null +++ b/pulp_docker/app/tasks/distribution.py @@ -0,0 +1,61 @@ +from django.core.exceptions import ObjectDoesNotExist + +from pulpcore.plugin.models import CreatedResource + +from pulp_docker.app.models import DockerDistribution +from pulp_docker.app.serializers import DockerDistributionSerializer + + +def create(*args, **kwargs): + """ + Creates a :class:`~pulp_docker.app.models.DockerDistribution`. + + Raises: + ValidationError: If the DockerDistributionSerializer is not valid + + """ + data = kwargs.pop('data', None) + serializer = DockerDistributionSerializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save() + resource = CreatedResource(content_object=serializer.instance) + resource.save() + + +def update(instance_id, *args, **kwargs): + """ + Updates a :class:`~pulp_docker.app.models.DockerDistribution`. + + Args: + instance_id (int): The id of the DockerDistribution to be updated + + Raises: + ValidationError: If the DistributionSerializer is not valid + + """ + data = kwargs.pop('data', None) + partial = kwargs.pop('partial', False) + instance = DockerDistribution.objects.get(pk=instance_id) + serializer = DockerDistributionSerializer(instance, data=data, partial=partial) + serializer.is_valid(raise_exception=True) + serializer.save() + + +def delete(instance_id, *args, **kwargs): + """ + Delete a :class:`~pulp_docker.app.models.DockerDistribution`. + + Args: + instance_id (int): The id of the DockerDistribution to be deleted + + Raises: + ObjectDoesNotExist: If the DockerDistribution was already deleted + + """ + try: + instance = DockerDistribution.objects.get(pk=instance_id) + except ObjectDoesNotExist: + # The object was already deleted, and we don't want an error thrown trying to delete again. + return + else: + instance.delete() diff --git a/pulp_docker/app/tasks/recursive_add.py b/pulp_docker/app/tasks/recursive_add.py new file mode 100644 index 00000000..837512e6 --- /dev/null +++ b/pulp_docker/app/tasks/recursive_add.py @@ -0,0 +1,73 @@ +from pulpcore.plugin.models import Repository, RepositoryVersion + +from pulp_docker.app.models import Blob, Manifest, MEDIA_TYPE, Tag + + +def recursive_add_content(repository_pk, content_units): + """ + Create a new repository version by recursively adding content. + + For each unit that is specified, we also need to add related content. For example, if a + manifest-list is specified, we need to add all referenced manifests, and all blobs referenced + by those manifests. + + Args: + repository_pk (int): The primary key for a Repository for which a new Repository Version + should be created. + content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that + should be added to the previous Repository Version for this Repository. + + """ + repository = Repository.objects.get(pk=repository_pk) + + tags_to_add = Tag.objects.filter( + pk__in=content_units, + ) + + manifest_lists_to_add = Manifest.objects.filter( + pk__in=content_units, + media_type=MEDIA_TYPE.MANIFEST_LIST + ) | Manifest.objects.filter( + pk__in=tags_to_add.values_list('tagged_manifest', flat=True), + media_type=MEDIA_TYPE.MANIFEST_LIST, + ) + + manifests_to_add = Manifest.objects.filter( + pk__in=content_units, + media_type__in=[MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V1_SIGNED, + MEDIA_TYPE.MANIFEST_V2] + ) | Manifest.objects.filter( + pk__in=manifest_lists_to_add.values_list('listed_manifests', flat=True) + ) | Manifest.objects.filter( + pk__in=tags_to_add.values_list('tagged_manifest', flat=True), + media_type__in=[MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V1_SIGNED, + MEDIA_TYPE.MANIFEST_V2] + ) + + blobs_to_add = Blob.objects.filter( + pk__in=content_units, + media_type__in=[MEDIA_TYPE.CONFIG_BLOB, MEDIA_TYPE.REGULAR_BLOB, MEDIA_TYPE.FOREIGN_BLOB] + ) | Blob.objects.filter( + pk__in=manifests_to_add.values_list('blobs', flat=True) + ) | Blob.objects.filter( + pk__in=manifests_to_add.values_list('config_blob', flat=True) + ) + + latest_version = RepositoryVersion.latest(repository) + if latest_version: + tags_in_repo = latest_version.content.filter( + pulp_type="docker.tag" + ) + tags_to_replace = Tag.objects.filter( + pk__in=tags_in_repo, + name__in=tags_to_add.values_list("name", flat=True) + ) + else: + tags_to_replace = [] + + with RepositoryVersion.create(repository) as new_version: + new_version.remove_content(tags_to_replace) + new_version.add_content(tags_to_add) + new_version.add_content(manifest_lists_to_add) + new_version.add_content(manifests_to_add) + new_version.add_content(blobs_to_add) diff --git a/pulp_docker/app/tasks/recursive_remove.py b/pulp_docker/app/tasks/recursive_remove.py new file mode 100644 index 00000000..59d917bd --- /dev/null +++ b/pulp_docker/app/tasks/recursive_remove.py @@ -0,0 +1,97 @@ +from django.db.models import Q +from pulpcore.plugin.models import Content, Repository, RepositoryVersion + +from pulp_docker.app.models import Blob, Manifest, MEDIA_TYPE, Tag + + +def recursive_remove_content(repository_pk, content_units): + """ + Create a new repository version by recursively removing content. + + For each unit that is specified, we also need to remove related content, + unless that content is also related to content that will remain in the + repository. For example, if a manifest-list is specified, we need to remove + all referenced manifests unless those manifests are referenced by a + manifest-list that will stay in the repository. + + For each content type, we identify 3 categories: + 1. must_remain: These content units are referenced by content units that will not be removed + 2. to_remove: These content units are either explicity given by the user, + or they are referenced by the content explicity given, and they are not in must_remain. + 3. to_remain: Content in the repo that is not in to_remove. This category + is used to determine must_remain of lower heirarchy content. + + + Args: + repository_pk (int): The primary key for a Repository for which a new Repository Version + should be created. + content_units (list): List of PKs for :class:`~pulpcore.app.models.Content` that + should be removed from the Repository. + + """ + repository = Repository.objects.get(pk=repository_pk) + latest_version = RepositoryVersion.latest(repository) + latest_content = latest_version.content.all() if latest_version else Content.objects.none() + + tags_in_repo = Q(pk__in=latest_content.filter(pulp_type='docker.tag')) + manifests_in_repo = Q(pk__in=latest_content.filter(pulp_type='docker.manifest')) + user_provided_content = Q(pk__in=content_units) + type_manifest_list = Q(media_type=MEDIA_TYPE.MANIFEST_LIST) + type_manifest = Q(media_type__in=[MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V2]) + blobs_in_repo = Q(pk__in=latest_content.filter(pulp_type='docker.blob')) + + # Tags do not have must_remain because they are the highest level content. + tags_to_remove = Tag.objects.filter(user_provided_content & tags_in_repo) + tags_to_remain = Tag.objects.filter(tags_in_repo).exclude(pk__in=tags_to_remove) + tagged_manifests_must_remain = Q( + pk__in=tags_to_remain.values_list("tagged_manifest", flat=True) + ) + tagged_manifests_to_remove = Q(pk__in=tags_to_remove.values_list("tagged_manifest", flat=True)) + + manifest_lists_must_remain = Manifest.objects.filter( + manifests_in_repo & tagged_manifests_must_remain & type_manifest_list + ) + manifest_lists_to_remove = Manifest.objects.filter( + user_provided_content | tagged_manifests_to_remove + ).filter( + type_manifest_list & manifests_in_repo + ).exclude(pk__in=manifest_lists_must_remain) + + manifest_lists_to_remain = Manifest.objects.filter( + manifests_in_repo & type_manifest_list + ).exclude(pk__in=manifest_lists_to_remove) + + listed_manifests_must_remain = Q( + pk__in=manifest_lists_to_remain.values_list('listed_manifests', flat=True) + ) + manifests_must_remain = Manifest.objects.filter( + tagged_manifests_must_remain | listed_manifests_must_remain + ).filter(type_manifest & manifests_in_repo) + + listed_manifests_to_remove = Q( + pk__in=manifest_lists_to_remove.values_list('listed_manifests', flat=True) + ) + manifests_to_remove = Manifest.objects.filter( + user_provided_content | listed_manifests_to_remove | tagged_manifests_to_remove + ).filter(type_manifest & manifests_in_repo).exclude(pk__in=manifests_must_remain) + + manifests_to_remain = Manifest.objects.filter( + manifests_in_repo & type_manifest + ).exclude(pk__in=manifests_to_remove) + + listed_blobs_must_remain = Q( + pk__in=manifests_to_remain.values_list('blobs', flat=True) + ) | Q(pk__in=manifests_to_remain.values_list('config_blob', flat=True)) + listed_blobs_to_remove = Q( + pk__in=manifests_to_remove.values_list('blobs', flat=True) + ) | Q(pk__in=manifests_to_remove.values_list('config_blob', flat=True)) + + blobs_to_remove = Blob.objects.filter( + user_provided_content | listed_blobs_to_remove + ).filter(blobs_in_repo).exclude(listed_blobs_must_remain) + + with RepositoryVersion.create(repository) as new_version: + new_version.remove_content(tags_to_remove) + new_version.remove_content(manifest_lists_to_remove) + new_version.remove_content(manifests_to_remove) + new_version.remove_content(blobs_to_remove) diff --git a/pulp_docker/app/tasks/sync_stages.py b/pulp_docker/app/tasks/sync_stages.py new file mode 100644 index 00000000..1acd60bf --- /dev/null +++ b/pulp_docker/app/tasks/sync_stages.py @@ -0,0 +1,520 @@ +import asyncio +import base64 +import json +import hashlib +import logging + +from gettext import gettext as _ +from urllib.parse import urljoin, urlparse, urlunparse + +from django.db import IntegrityError +from pulpcore.plugin.models import Artifact, ProgressReport, Remote +from pulpcore.plugin.stages import DeclarativeArtifact, DeclarativeContent, Stage +from pulpcore.constants import TASK_STATES + +from pulp_docker.app.models import (Manifest, MEDIA_TYPE, Blob, Tag, + BlobManifest, ManifestListManifest) + + +log = logging.getLogger(__name__) + + +V2_ACCEPT_HEADERS = { + 'Accept': ','.join([MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_LIST]) +} + + +class DockerFirstStage(Stage): + """ + The first stage of a pulp_docker sync pipeline. + + In this stage all the content is discovered, including the nested one. + + """ + + def __init__(self, remote): + """Initialize the stage.""" + super().__init__() + self.remote = remote + self.deferred_download = (self.remote.policy != Remote.IMMEDIATE) + + async def run(self): + """ + DockerFirstStage. + """ + future_manifests = [] + tag_list = [] + to_download = [] + man_dcs = {} + total_blobs = [] + + with ProgressReport( + message='Downloading tag list', code='downloading.tag_list', total=1 + ) as pb: + repo_name = self.remote.namespaced_upstream_name + relative_url = '/v2/{name}/tags/list'.format(name=repo_name) + tag_list_url = urljoin(self.remote.url, relative_url) + list_downloader = self.remote.get_downloader(url=tag_list_url) + await list_downloader.run(extra_data={'repo_name': repo_name}) + + with open(list_downloader.path) as tags_raw: + tags_dict = json.loads(tags_raw.read()) + tag_list = tags_dict['tags'] + + # check for the presence of the pagination link header + link = list_downloader.response_headers.get('Link') + await self.handle_pagination(link, repo_name, tag_list) + whitelist_tags = self.remote.whitelist_tags + if whitelist_tags: + tag_list = list(set(tag_list) & set(whitelist_tags)) + pb.increment() + + for tag_name in tag_list: + relative_url = '/v2/{name}/manifests/{tag}'.format( + name=self.remote.namespaced_upstream_name, + tag=tag_name, + ) + url = urljoin(self.remote.url, relative_url) + downloader = self.remote.get_downloader(url=url) + to_download.append(downloader.run(extra_data={'headers': V2_ACCEPT_HEADERS})) + + pb_parsed_tags = ProgressReport( + message='Processing Tags', + code='processing.tag', + state=TASK_STATES.RUNNING, + total=len(tag_list) + ) + + for download_tag in asyncio.as_completed(to_download): + tag = await download_tag + with open(tag.path, 'rb') as content_file: + raw_data = content_file.read() + content_data = json.loads(raw_data) + media_type = content_data.get('mediaType') + tag.artifact_attributes['file'] = tag.path + saved_artifact = Artifact(**tag.artifact_attributes) + try: + saved_artifact.save() + except IntegrityError: + del tag.artifact_attributes['file'] + saved_artifact = Artifact.objects.get(**tag.artifact_attributes) + tag_dc = self.create_tag(saved_artifact, tag.url) + + if media_type == MEDIA_TYPE.MANIFEST_LIST: + list_dc = self.create_tagged_manifest_list( + tag_dc, content_data) + await self.put(list_dc) + tag_dc.extra_data['man_relation'] = list_dc + for manifest_data in content_data.get('manifests'): + man_dc = self.create_manifest(list_dc, manifest_data) + future_manifests.append(man_dc.get_or_create_future()) + man_dcs[man_dc.content.digest] = man_dc + await self.put(man_dc) + else: + man_dc = self.create_tagged_manifest(tag_dc, content_data, raw_data) + await self.put(man_dc) + tag_dc.extra_data['man_relation'] = man_dc + self.handle_blobs(man_dc, content_data, total_blobs) + await self.put(tag_dc) + pb_parsed_tags.increment() + + pb_parsed_tags.state = 'completed' + pb_parsed_tags.save() + + for manifest_future in asyncio.as_completed(future_manifests): + man = await manifest_future + with man._artifacts.get().file.open() as content_file: + raw = content_file.read() + content_data = json.loads(raw) + man_dc = man_dcs[man.digest] + self.handle_blobs(man_dc, content_data, total_blobs) + for blob in total_blobs: + await self.put(blob) + + async def handle_pagination(self, link, repo_name, tag_list): + """ + Handle registries that have pagination enabled. + """ + while link: + # according RFC5988 URI-reference can be relative or absolute + _, _, path, params, query, fragm = urlparse(link.split(';')[0].strip('>, <')) + rel_link = urlunparse(('', '', path, params, query, fragm)) + link = urljoin(self.remote.url, rel_link) + list_downloader = self.remote.get_downloader(url=link) + await list_downloader.run(extra_data={'repo_name': repo_name}) + with open(list_downloader.path) as tags_raw: + tags_dict = json.loads(tags_raw.read()) + tag_list.extend(tags_dict['tags']) + link = list_downloader.response_headers.get('Link') + + def handle_blobs(self, man, content_data, total_blobs): + """ + Handle blobs. + """ + for layer in (content_data.get("layers") or content_data.get("fsLayers")): + if not self._include_layer(layer): + continue + blob_dc = self.create_blob(man, layer) + blob_dc.extra_data['blob_relation'] = man + total_blobs.append(blob_dc) + layer = content_data.get('config', None) + if layer: + blob_dc = self.create_blob(man, layer) + blob_dc.extra_data['config_relation'] = man + total_blobs.append(blob_dc) + + def create_tag(self, saved_artifact, url): + """ + Create `DeclarativeContent` for each tag. + + Each dc contains enough information to be dowloaded by an ArtifactDownload Stage. + + Args: + tag_name (str): Name of each tag + + Returns: + pulpcore.plugin.stages.DeclarativeContent: A Tag DeclarativeContent object + + """ + tag_name = url.split('/')[-1] + relative_url = '/v2/{name}/manifests/{tag}'.format( + name=self.remote.namespaced_upstream_name, + tag=tag_name, + ) + url = urljoin(self.remote.url, relative_url) + tag = Tag(name=tag_name) + da = DeclarativeArtifact( + artifact=saved_artifact, + url=url, + relative_path=tag_name, + remote=self.remote, + extra_data={'headers': V2_ACCEPT_HEADERS} + ) + tag_dc = DeclarativeContent(content=tag, d_artifacts=[da]) + return tag_dc + + def create_tagged_manifest_list(self, tag_dc, manifest_list_data): + """ + Create a ManifestList. + + Args: + tag_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Tag + manifest_list_data (dict): Data about a ManifestList + + """ + digest = "sha256:{digest}".format(digest=tag_dc.d_artifacts[0].artifact.sha256) + relative_url = '/v2/{name}/manifests/{digest}'.format( + name=self.remote.namespaced_upstream_name, + digest=digest, + ) + url = urljoin(self.remote.url, relative_url) + manifest_list = Manifest( + digest=digest, + schema_version=manifest_list_data['schemaVersion'], + media_type=manifest_list_data['mediaType'], + ) + da = DeclarativeArtifact( + artifact=tag_dc.d_artifacts[0].artifact, + url=url, + relative_path=digest, + remote=self.remote, + extra_data={'headers': V2_ACCEPT_HEADERS} + ) + list_dc = DeclarativeContent(content=manifest_list, d_artifacts=[da]) + + return list_dc + + def create_tagged_manifest(self, tag_dc, manifest_data, raw_data): + """ + Create an Image Manifest. + + Args: + tag_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Tag + manifest_data (dict): Data about a single new ImageManifest. + raw_data: (str): The raw JSON representation of the ImageManifest. + + """ + media_type = manifest_data.get('mediaType', MEDIA_TYPE.MANIFEST_V1) + if media_type == MEDIA_TYPE.MANIFEST_V2: + digest = "sha256:{digest}".format(digest=tag_dc.d_artifacts[0].artifact.sha256) + else: + + digest = self._calculate_digest(raw_data) + manifest = Manifest( + digest=digest, + schema_version=manifest_data['schemaVersion'], + media_type=media_type + ) + relative_url = '/v2/{name}/manifests/{digest}'.format( + name=self.remote.namespaced_upstream_name, + digest=digest, + ) + url = urljoin(self.remote.url, relative_url) + da = DeclarativeArtifact( + artifact=tag_dc.d_artifacts[0].artifact, + url=url, + relative_path=digest, + remote=self.remote, + extra_data={'headers': V2_ACCEPT_HEADERS} + ) + man_dc = DeclarativeContent(content=manifest, d_artifacts=[da]) + return man_dc + + def create_manifest(self, list_dc, manifest_data): + """ + Create an Image Manifest from manifest data in a ManifestList. + + Args: + list_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ManifestList + manifest_data (dict): Data about a single new ImageManifest. + + """ + digest = manifest_data['digest'] + relative_url = '/v2/{name}/manifests/{digest}'.format( + name=self.remote.namespaced_upstream_name, + digest=digest + ) + manifest_url = urljoin(self.remote.url, relative_url) + da = DeclarativeArtifact( + artifact=Artifact(), + url=manifest_url, + relative_path=digest, + remote=self.remote, + extra_data={'headers': V2_ACCEPT_HEADERS} + ) + manifest = Manifest( + digest=manifest_data['digest'], + schema_version=2 if manifest_data['mediaType'] == MEDIA_TYPE.MANIFEST_V2 else 1, + media_type=manifest_data['mediaType'], + ) + platform = {} + p = manifest_data['platform'] + platform['architecture'] = p['architecture'] + platform['os'] = p['os'] + platform['features'] = p.get('features', '') + platform['variant'] = p.get('variant', '') + platform['os.version'] = p.get('os.version', '') + platform['os.features'] = p.get('os.features', '') + man_dc = DeclarativeContent( + content=manifest, + d_artifacts=[da], + extra_data={'relation': list_dc, 'platform': platform}, + does_batch=False, + ) + return man_dc + + def create_blob(self, man_dc, blob_data): + """ + Create blob. + + Args: + man_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ImageManifest + blob_data (dict): Data about a blob + + """ + digest = blob_data.get('digest') or blob_data.get('blobSum') + blob_artifact = Artifact(sha256=digest[len("sha256:"):]) + blob = Blob( + digest=digest, + media_type=blob_data.get('mediaType', MEDIA_TYPE.REGULAR_BLOB), + ) + relative_url = '/v2/{name}/blobs/{digest}'.format( + name=self.remote.namespaced_upstream_name, + digest=digest, + ) + blob_url = urljoin(self.remote.url, relative_url) + da = DeclarativeArtifact( + artifact=blob_artifact, + url=blob_url, + relative_path=digest, + remote=self.remote, + extra_data={'headers': V2_ACCEPT_HEADERS}, + deferred_download=self.deferred_download + ) + blob_dc = DeclarativeContent( + content=blob, + d_artifacts=[da], + ) + + return blob_dc + + def _include_layer(self, layer): + """ + Decide whether to include a layer. + + Args: + layer (dict): Layer reference. + + Returns: + bool: True when the layer should be included. + + """ + foreign_excluded = (not self.remote.include_foreign_layers) + is_foreign = (layer.get('mediaType', MEDIA_TYPE.REGULAR_BLOB) == MEDIA_TYPE.FOREIGN_BLOB) + if is_foreign and foreign_excluded: + log.debug(_('Foreign Layer: %(d)s EXCLUDED'), dict(d=layer)) + return False + return True + + def _calculate_digest(self, manifest): + """ + Calculate the requested digest of the ImageManifest, given in JSON. + + Args: + manifest (str): The raw JSON representation of the Manifest. + + Returns: + str: The digest of the given ImageManifest + + """ + decoded_manifest = json.loads(manifest) + if 'signatures' in decoded_manifest: + # This manifest contains signatures. Unfortunately, the Docker manifest digest + # is calculated on the unsigned version of the Manifest so we need to remove the + # signatures. To do this, we will look at the 'protected' key within the first + # signature. This key indexes a (malformed) base64 encoded JSON dictionary that + # tells us how many bytes of the manifest we need to keep before the signature + # appears in the original JSON and what the original ending to the manifest was after + # the signature block. We will strip out the bytes after this cutoff point, add back the + # original ending, and then calculate the sha256 sum of the transformed JSON to get the + # digest. + protected = decoded_manifest['signatures'][0]['protected'] + # Add back the missing padding to the protected block so that it is valid base64. + protected = self._pad_unpadded_b64(protected) + # Now let's decode the base64 and load it as a dictionary so we can get the length + protected = base64.b64decode(protected) + protected = json.loads(protected) + # This is the length of the signed portion of the Manifest, except for a trailing + # newline and closing curly brace. + signed_length = protected['formatLength'] + # The formatTail key indexes a base64 encoded string that represents the end of the + # original Manifest before signatures. We will need to add this string back to the + # trimmed Manifest to get the correct digest. We'll do this as a one liner since it is + # a very similar process to what we've just done above to get the protected block + # decoded. + signed_tail = base64.b64decode(self._pad_unpadded_b64(protected['formatTail'])) + # Now we can reconstruct the original Manifest that the digest should be based on. + manifest = manifest[:signed_length] + signed_tail + + return "sha256:{digest}".format(digest=hashlib.sha256(manifest).hexdigest()) + + def _pad_unpadded_b64(self, unpadded_b64): + """ + Fix bad padding. + + Docker has not included the required padding at the end of the base64 encoded + 'protected' block, or in some encased base64 within it. This function adds the correct + number of ='s signs to the unpadded base64 text so that it can be decoded with Python's + base64 library. + + Args: + unpadded_b64 (str): The unpadded base64 text. + + Returns: + str: The same base64 text with the appropriate number of ='s symbols. + + """ + # The Pulp team has not observed any newlines or spaces within the base64 from Docker, but + # Docker's own code does this same operation so it seemed prudent to include it here. + # See lines 167 to 168 here: + # https://github.com/docker/libtrust/blob/9cbd2a1374f46905c68a4eb3694a130610adc62a/util.go + unpadded_b64 = unpadded_b64.replace('\n', '').replace(' ', '') + # It is illegal base64 for the remainder to be 1 when the length of the block is + # divided by 4. + if len(unpadded_b64) % 4 == 1: + raise ValueError(_('Invalid base64: {t}').format(t=unpadded_b64)) + # Add back the missing padding characters, based on the length of the encoded string + paddings = {0: '', 2: '==', 3: '='} + return unpadded_b64 + paddings[len(unpadded_b64) % 4] + + +class InterrelateContent(Stage): + """ + Stage for relating Content to other Content. + """ + + async def run(self): + """ + Relate each item in the input queue to objects specified on the DeclarativeContent. + """ + async for dc in self.items(): + + if dc.extra_data.get('relation'): + self.relate_manifest_to_list(dc) + elif dc.extra_data.get('blob_relation'): + self.relate_blob(dc) + elif dc.extra_data.get('config_relation'): + self.relate_config_blob(dc) + elif dc.extra_data.get('man_relation'): + self.relate_manifest_tag(dc) + + await self.put(dc) + + def relate_config_blob(self, dc): + """ + Relate a Blob to a Manifest as a config layer. + + Args: + dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Blob + + """ + configured_dc = dc.extra_data.get('config_relation') + configured_dc.content.config_blob = dc.content + configured_dc.content.save() + + def relate_blob(self, dc): + """ + Relate a Blob to a Manifest. + + Args: + dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Blob + + """ + related_dc = dc.extra_data.get('blob_relation') + thru = BlobManifest(manifest=related_dc.content, manifest_blob=dc.content) + try: + thru.save() + except IntegrityError: + pass + + def relate_manifest_tag(self, dc): + """ + Relate an ImageManifest to a Tag. + + Args: + dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Tag + + """ + related_dc = dc.extra_data.get('man_relation') + dc.content.tagged_manifest = related_dc.content + try: + dc.content.save() + except IntegrityError: + existing_tag = Tag.objects.get(name=dc.content.name, + tagged_manifest=related_dc.content) + dc.content.delete() + dc.content = existing_tag + + def relate_manifest_to_list(self, dc): + """ + Relate an ImageManifest to a ManifestList. + + Args: + dc (pulpcore.plugin.stages.DeclarativeContent): dc for a ImageManifest + + """ + related_dc = dc.extra_data.get('relation') + platform = dc.extra_data.get('platform') + thru = ManifestListManifest(manifest_list=dc.content, image_manifest=related_dc.content, + architecture=platform['architecture'], + os=platform['os'], + features=platform.get('features'), + variant=platform.get('variant'), + os_version=platform.get('os.version'), + os_features=platform.get('os.features') + ) + + try: + thru.save() + except IntegrityError: + pass diff --git a/pulp_docker/app/tasks/synchronize.py b/pulp_docker/app/tasks/synchronize.py new file mode 100644 index 00000000..e25fcdfb --- /dev/null +++ b/pulp_docker/app/tasks/synchronize.py @@ -0,0 +1,83 @@ +from gettext import gettext as _ +import logging + +from pulpcore.plugin.models import Repository +from pulpcore.plugin.stages import ( + ArtifactDownloader, + ArtifactSaver, + ContentSaver, + DeclarativeVersion, + RemoteArtifactSaver, + RemoveDuplicates, + ResolveContentFutures, + QueryExistingArtifacts, + QueryExistingContents, +) + +from .sync_stages import InterrelateContent, DockerFirstStage +from pulp_docker.app.models import DockerRemote, Tag + + +log = logging.getLogger(__name__) + + +def synchronize(remote_pk, repository_pk): + """ + Sync content from the remote repository. + + Create a new version of the repository that is synchronized with the remote. + + Args: + remote_pk (str): The remote PK. + repository_pk (str): The repository PK. + + Raises: + ValueError: If the remote does not specify a URL to sync + + """ + remote = DockerRemote.objects.get(pk=remote_pk) + repository = Repository.objects.get(pk=repository_pk) + if not remote.url: + raise ValueError(_('A remote must have a url specified to synchronize.')) + remove_duplicate_tags = [{'model': Tag, 'field_names': ['name']}] + log.info(_('Synchronizing: repository={r} remote={p}').format( + r=repository.name, p=remote.name)) + first_stage = DockerFirstStage(remote) + dv = DockerDeclarativeVersion(first_stage, repository, remove_duplicates=remove_duplicate_tags) + dv.create() + + +class DockerDeclarativeVersion(DeclarativeVersion): + """ + Subclassed Declarative version creates a custom pipeline for Docker sync. + """ + + def pipeline_stages(self, new_version): + """ + Build a list of stages feeding into the ContentUnitAssociation stage. + + This defines the "architecture" of the entire sync. + + Args: + new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The + new repository version that is going to be built. + + Returns: + list: List of :class:`~pulpcore.plugin.stages.Stage` instances + + """ + pipeline = [ + self.first_stage, + QueryExistingArtifacts(), + ArtifactDownloader(), + ArtifactSaver(), + QueryExistingContents(), + ContentSaver(), + RemoteArtifactSaver(), + ResolveContentFutures(), + InterrelateContent(), + ] + for dupe_query_dict in self.remove_duplicates: + pipeline.append(RemoveDuplicates(new_version, **dupe_query_dict)) + + return pipeline diff --git a/pulp_docker/app/tasks/tag.py b/pulp_docker/app/tasks/tag.py new file mode 100644 index 00000000..f1a9d806 --- /dev/null +++ b/pulp_docker/app/tasks/tag.py @@ -0,0 +1,51 @@ +from pulpcore.plugin.models import Repository, RepositoryVersion, ContentArtifact, CreatedResource +from pulp_docker.app.models import Manifest, Tag + + +def tag_image(manifest_pk, tag, repository_pk): + """ + Create a new repository version out of the passed tag name and the manifest. + + If the tag name is already associated with an existing manifest with the same digest, + no new content is created. Note that a same tag name cannot be used for two different + manifests. Due to this fact, an old Tag object is going to be removed from + a new repository version when a manifest contains a digest which is not equal to the + digest passed with POST request. + """ + manifest = Manifest.objects.get(pk=manifest_pk) + artifact = manifest._artifacts.all()[0] + + repository = Repository.objects.get(pk=repository_pk) + latest_version = RepositoryVersion.latest(repository) + + tags_to_remove = Tag.objects.filter( + pk__in=latest_version.content.all(), + name=tag + ).exclude( + tagged_manifest=manifest + ) + + manifest_tag, created = Tag.objects.get_or_create( + name=tag, + tagged_manifest=manifest + ) + + if created: + resource = CreatedResource(content_object=manifest_tag) + resource.save() + + ContentArtifact.objects.get_or_create( + artifact=artifact, + content=manifest_tag, + relative_path=tag + ) + + tags_to_add = Tag.objects.filter( + pk=manifest_tag.pk + ).exclude( + pk__in=latest_version.content.all() + ) + + with RepositoryVersion.create(repository) as repository_version: + repository_version.remove_content(tags_to_remove) + repository_version.add_content(tags_to_add) diff --git a/pulp_docker/app/tasks/untag.py b/pulp_docker/app/tasks/untag.py new file mode 100644 index 00000000..55fd9992 --- /dev/null +++ b/pulp_docker/app/tasks/untag.py @@ -0,0 +1,22 @@ +from pulpcore.plugin.models import Repository, RepositoryVersion +from pulp_docker.app.models import Tag + + +def untag_image(tag, repository_pk): + """ + Create a new repository version without a specified manifest's tag name. + """ + repository = Repository.objects.get(pk=repository_pk) + latest_version = RepositoryVersion.latest(repository) + + tags_in_latest_repository = latest_version.content.filter( + pulp_type="docker.tag" + ) + + tags_to_remove = Tag.objects.filter( + pk__in=tags_in_latest_repository, + name=tag + ) + + with RepositoryVersion.create(repository) as repository_version: + repository_version.remove_content(tags_to_remove) diff --git a/pulp_docker/app/token_verification.py b/pulp_docker/app/token_verification.py new file mode 100644 index 00000000..c74228a4 --- /dev/null +++ b/pulp_docker/app/token_verification.py @@ -0,0 +1,177 @@ +import jwt + +from aiohttp import web +from django.conf import settings + + +class TokenVerifier: + """A class used for a token verification.""" + + def __init__(self, request, access_action): + """ + Store data required for the token verification. + + Args: + request (:class:`~aiohttp.web.Request`): The request with an Authorization header. + access_action (str): A required action to perform pulling/pushing. + + """ + self.request = request + self.access_action = access_action + + def verify(self): + """Verify a Bearer token.""" + authorization = self.get_authorization_header() + self.check_authorization_token(authorization) + + def get_authorization_header(self): + """ + Fetch an Authorization header from the request. + + Raises: + web.HTTPUnauthorized: An Authorization header is missing in the header. + + Returns: + A raw string containing a Bearer token. + + """ + try: + return self.request.headers['Authorization'] + except KeyError: + raise web.HTTPUnauthorized( + headers=self._build_response_headers(), + reason='Access to the requested resource is not authorized. ' + 'A Bearer token is missing in a request header.' + ) + + def check_authorization_token(self, authorization): + """ + Check if a Bearer token is valid. + + Args: + authorization: A string containing a Bearer token. + + Raises: + web.HTTPUnauthorized: A Bearer token is not valid. + + """ + token = self._get_token(authorization) + if not self.is_token_valid(token): + raise web.HTTPUnauthorized( + headers=self._build_response_headers(), + reason='Access to the requested resource is not authorized. ' + 'A provided Bearer token is invalid.' + ) + + def _get_token(self, authorization): + """ + Get a raw token string from the header. + + This method returns a string that skips the keyword 'Bearer' and an additional + space from the header (e.g. "Bearer abcdef123456" -> "abcdef123456"). + """ + return authorization[7:] + + def _build_response_headers(self): + """ + Build headers that a registry returns as a response to unauthorized access. + + The method creates a value for the Www-Authenticate header. This value is used + by a client for requesting a Bearer token from a token server. The header + Docker-Distribution-API-Version is generated too to inform the client about + the supported schema type. + """ + source_path = self._get_current_content_path() + authenticate_header = self._build_authenticate_string(source_path) + + headers = { + 'Docker-Distribution-API-Version': 'registry/2.0', + 'Www-Authenticate': authenticate_header + } + return headers + + def _build_authenticate_string(self, source_path): + """ + Build a formatted authenticate string. + + For example, A created string is the following format: + realm="https://token",service="docker.io",scope="repository:my-app:push". + """ + realm = f'{self.request.scheme}://{settings.TOKEN_SERVER}' + authenticate_string = f'Bearer realm="{realm}",service="{settings.CONTENT_ORIGIN}"' + + if not self._is_verifying_root_endpoint(): + scope = f'repository:{source_path}:pull' + authenticate_string += f',scope="{scope}"' + + return authenticate_string + + def is_token_valid(self, encoded_token): + """Decode and validate a token.""" + with open(settings.PUBLIC_KEY_PATH, 'rb') as public_key: + decoded_token = self.decode_token(encoded_token, public_key.read()) + + return self.contains_accessible_actions(decoded_token) + + def decode_token(self, encoded_token, public_key): + """ + Decode token and verify a signature with a public key. + + If the token could not be decoded with a success, a client does not have a + permission to operate with a registry. + """ + jwt_config = self._init_jwt_decoder_config() + try: + decoded_token = jwt.decode(encoded_token, public_key, **jwt_config) + except jwt.exceptions.InvalidTokenError: + decoded_token = {'access': []} + return decoded_token + + def _init_jwt_decoder_config(self): + """Initialize a basic configuration used for sanitizing and decoding a token.""" + return { + 'algorithms': [settings.TOKEN_SIGNATURE_ALGORITHM], + 'issuer': settings.TOKEN_SERVER, + 'audience': settings.CONTENT_ORIGIN + } + + def contains_accessible_actions(self, decoded_token): + """Check if a client has an access permission to execute the pull/push operation.""" + for access in decoded_token['access']: + if self._targets_current_content_path(access): + return True + + return False + + def _targets_current_content_path(self, access): + """ + Check if a client targets a valid content path. + + When a client targets the root endpoint, the verifier does not necessary need to + check for the pull or push access permission, therefore, it is granted automatically. + """ + content_path = self._get_current_content_path() + + if content_path == access['name']: + if self.access_action in access['actions']: + return True + if self._is_verifying_root_endpoint(): + return True + + return False + + def _get_current_content_path(self): + """ + Retrieve a content path from the request. + + If the path does not exist, it means that a client is querying a root endpoint. + """ + try: + content_path = self.request.match_info['path'] + except KeyError: + content_path = '' + return content_path + + def _is_verifying_root_endpoint(self): + """If the root endpoint is queried, no matching info is present.""" + return not bool(self.request.match_info) diff --git a/pulp_docker/app/urls.py b/pulp_docker/app/urls.py new file mode 100644 index 00000000..0f620a0a --- /dev/null +++ b/pulp_docker/app/urls.py @@ -0,0 +1,20 @@ +from django.conf.urls import url + +from .viewsets import ( + ManifestCopyViewSet, + RecursiveAdd, + RecursiveRemove, + TagCopyViewSet, + TagImageViewSet, + UnTagImageViewSet, +) + + +urlpatterns = [ + url(r'^pulp/api/v3/docker/manifests/copy/$', ManifestCopyViewSet.as_view({'post': 'create'})), + url(r'^pulp/api/v3/docker/recursive-add/$', RecursiveAdd.as_view({'post': 'create'})), + url(r'^pulp/api/v3/docker/recursive-remove/$', RecursiveRemove.as_view({'post': 'create'})), + url(r'^pulp/api/v3/docker/tag/$', TagImageViewSet.as_view({'post': 'create'})), + url(r'^pulp/api/v3/docker/tags/copy/$', TagCopyViewSet.as_view({'post': 'create'})), + url(r'^pulp/api/v3/docker/untag/$', UnTagImageViewSet.as_view({'post': 'create'})) +] diff --git a/pulp_docker/app/viewsets.py b/pulp_docker/app/viewsets.py new file mode 100644 index 00000000..e2fddbea --- /dev/null +++ b/pulp_docker/app/viewsets.py @@ -0,0 +1,394 @@ +""" +Check `Plugin Writer's Guide`_ for more details. + +. _Plugin Writer's Guide: + http://docs.pulpproject.org/en/3.0/nightly/plugins/plugin-writer/index.html +""" + +from django_filters import MultipleChoiceFilter +from drf_yasg.utils import swagger_auto_schema +from pulpcore.plugin.serializers import ( + AsyncOperationResponseSerializer, + RepositorySyncURLSerializer, +) +from pulpcore.plugin.models import Content +from pulpcore.plugin.tasking import enqueue_with_reservation +from pulpcore.plugin.viewsets import ( + BaseDistributionViewSet, + CharInFilter, + ContentFilter, + NamedModelViewSet, + ReadOnlyContentViewSet, + RemoteViewSet, + OperationPostponedResponse, +) +from rest_framework import viewsets as drf_viewsets +from rest_framework.decorators import action + +from . import models, serializers, tasks + + +class TagFilter(ContentFilter): + """ + FilterSet for Tags. + """ + + media_type = MultipleChoiceFilter( + choices=models.Manifest.MANIFEST_CHOICES, + field_name='tagged_manifest__media_type', + lookup_expr='contains', + ) + digest = CharInFilter(field_name='tagged_manifest__digest', lookup_expr='in') + + class Meta: + model = models.Tag + fields = { + 'name': ['exact', 'in'], + } + + +class ManifestFilter(ContentFilter): + """ + FilterSet for Manifests. + """ + + media_type = MultipleChoiceFilter(choices=models.Manifest.MANIFEST_CHOICES) + + class Meta: + model = models.Manifest + fields = { + 'digest': ['exact', 'in'], + } + + +class TagViewSet(ReadOnlyContentViewSet): + """ + ViewSet for Tag. + """ + + endpoint_name = 'tags' + queryset = models.Tag.objects.all() + serializer_class = serializers.TagSerializer + filterset_class = TagFilter + + +class ManifestViewSet(ReadOnlyContentViewSet): + """ + ViewSet for Manifest. + """ + + endpoint_name = 'manifests' + queryset = models.Manifest.objects.all() + serializer_class = serializers.ManifestSerializer + filterset_class = ManifestFilter + + +class BlobFilter(ContentFilter): + """ + FilterSet for Blobs. + """ + + media_type = MultipleChoiceFilter(choices=models.Blob.BLOB_CHOICES) + + class Meta: + model = models.Blob + fields = { + 'digest': ['exact', 'in'], + } + + +class BlobViewSet(ReadOnlyContentViewSet): + """ + ViewSet for Blobs. + """ + + endpoint_name = 'blobs' + queryset = models.Blob.objects.all() + serializer_class = serializers.BlobSerializer + filterset_class = BlobFilter + + +class DockerRemoteViewSet(RemoteViewSet): + """ + Docker remotes represent an external repository that implements the Docker + Registry API. Docker remotes support deferred downloading by configuring + the ``policy`` field. ``on_demand`` and ``streamed`` policies can provide + significant disk space savings. + """ + + endpoint_name = 'docker' + queryset = models.DockerRemote.objects.all() + serializer_class = serializers.DockerRemoteSerializer + + # This decorator is necessary since a sync operation is asyncrounous and returns + # the id and href of the sync task. + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to sync content.", + responses={202: AsyncOperationResponseSerializer} + ) + @action(detail=True, methods=['post'], serializer_class=RepositorySyncURLSerializer) + def sync(self, request, pk): + """ + Synchronizes a repository. The ``repository`` field has to be provided. + """ + remote = self.get_object() + serializer = RepositorySyncURLSerializer(data=request.data, context={'request': request}) + + # Validate synchronously to return 400 errors. + serializer.is_valid(raise_exception=True) + repository = serializer.validated_data.get('repository') + result = enqueue_with_reservation( + tasks.synchronize, + [repository, remote], + kwargs={ + 'remote_pk': remote.pk, + 'repository_pk': repository.pk + } + ) + return OperationPostponedResponse(result, request) + + +class DockerDistributionViewSet(BaseDistributionViewSet): + """ + The Docker Distribution will serve the latest version of a Repository if + ``repository`` is specified. The Docker Distribution will serve a specific + repository version if ``repository_version``. Note that **either** + ``repository`` or ``repository_version`` can be set on a Docker + Distribution, but not both. + """ + + endpoint_name = 'docker' + queryset = models.DockerDistribution.objects.all() + serializer_class = serializers.DockerDistributionSerializer + + +class TagImageViewSet(drf_viewsets.ViewSet): + """ + ViewSet used for tagging manifests. This endpoint supports only HTTP POST requests. + """ + + endpoint_name = 'tag' + + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to create a new repository", + responses={202: AsyncOperationResponseSerializer}, + request_body=serializers.TagImageSerializer, + ) + def create(self, request): + """ + Create a task which is responsible for initializing a new repository version. + """ + serializer = serializers.TagImageSerializer( + data=request.data, + context={'request': request} + ) + serializer.is_valid(raise_exception=True) + + manifest = serializer.validated_data['manifest'] + tag = serializer.validated_data['tag'] + repository = serializer.validated_data['repository'] + + result = enqueue_with_reservation( + tasks.tag_image, + [repository, manifest], + kwargs={ + 'manifest_pk': manifest.pk, + 'tag': tag, + 'repository_pk': repository.pk + } + ) + return OperationPostponedResponse(result, request) + + +class UnTagImageViewSet(drf_viewsets.ViewSet): + """ + ViewSet used for untagging manifests. This endpoint supports only HTTP POST requests. + """ + + endpoint_name = 'untag' + + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to create a new repository", + responses={202: AsyncOperationResponseSerializer}, + request_body=serializers.UnTagImageSerializer, + ) + def create(self, request): + """ + Create a task which is responsible for creating a new tag. + """ + serializer = serializers.UnTagImageSerializer( + data=request.data, + context={'request': request} + ) + serializer.is_valid(raise_exception=True) + + tag = serializer.validated_data['tag'] + repository = serializer.validated_data['repository'] + + result = enqueue_with_reservation( + tasks.untag_image, + [repository], + kwargs={ + 'tag': tag, + 'repository_pk': repository.pk + } + ) + return OperationPostponedResponse(result, request) + + +class RecursiveAdd(drf_viewsets.ViewSet): + """ + ViewSet for recursively adding Docker content. + """ + + serializer_class = serializers.RecursiveManageSerializer + + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to recursively add docker content.", + responses={202: AsyncOperationResponseSerializer}, + request_body=serializers.RecursiveManageSerializer, + ) + def create(self, request): + """ + Queues a task that creates a new RepositoryVersion by adding content units. + """ + add_content_units = [] + serializer = serializers.RecursiveManageSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + repository = serializer.validated_data['repository'] + + if 'content_units' in request.data: + for url in request.data['content_units']: + content = NamedModelViewSet.get_resource(url, Content) + add_content_units.append(content.pk) + + result = enqueue_with_reservation( + tasks.recursive_add_content, [repository], + kwargs={ + 'repository_pk': repository.pk, + 'content_units': add_content_units, + } + ) + return OperationPostponedResponse(result, request) + + +class TagCopyViewSet(drf_viewsets.ViewSet): + """ + ViewSet for copying tags recursively. + """ + + serializer_class = serializers.TagCopySerializer + + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to copy tags", + responses={202: AsyncOperationResponseSerializer}, + request_body=serializers.TagCopySerializer, + ) + def create(self, request): + """ + Queues a task that creates a new RepositoryVersion by adding content units. + """ + names = request.data.get("names") + serializer = serializers.TagCopySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + source_latest = serializer.validated_data['source_repository_version'] + destination = serializer.validated_data['destination_repository'] + content_tags_in_repo = source_latest.content.filter( + pulp_type="docker.tag" + ) + tags_in_repo = models.Tag.objects.filter( + pk__in=content_tags_in_repo, + ) + if names is None: + tags_to_add = tags_in_repo + else: + tags_to_add = tags_in_repo.filter(name__in=names) + + result = enqueue_with_reservation( + tasks.recursive_add_content, [destination], + kwargs={ + 'repository_pk': destination.pk, + 'content_units': tags_to_add.values_list('pk', flat=True), + } + ) + return OperationPostponedResponse(result, request) + + +class ManifestCopyViewSet(drf_viewsets.ViewSet): + """ + ViewSet for copying manifests recursively. + """ + + serializer_class = serializers.ManifestCopySerializer + + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to copy manifests", + responses={202: AsyncOperationResponseSerializer}, + request_body=serializers.ManifestCopySerializer, + ) + def create(self, request): + """ + Queues a task that creates a new RepositoryVersion by adding content units. + """ + serializer = serializers.ManifestCopySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + source_latest = serializer.validated_data['source_repository_version'] + destination = serializer.validated_data['destination_repository'] + content_manifests_in_repo = source_latest.content.filter( + pulp_type="docker.manifest" + ) + manifests_in_repo = models.Manifest.objects.filter( + pk__in=content_manifests_in_repo, + ) + digests = request.data.get("digests") + media_types = request.data.get("media_types") + filters = {} + if digests is not None: + filters['digest__in'] = digests + if media_types is not None: + filters['media_type__in'] = media_types + manifests_to_add = manifests_in_repo.filter(**filters) + result = enqueue_with_reservation( + tasks.recursive_add_content, [destination], + kwargs={ + 'repository_pk': destination.pk, + 'content_units': manifests_to_add, + } + ) + return OperationPostponedResponse(result, request) + + +class RecursiveRemove(drf_viewsets.ViewSet): + """ + ViewSet for recursively removing Docker content. + """ + + serializer_class = serializers.RecursiveManageSerializer + + @swagger_auto_schema( + operation_description="Trigger an asynchronous task to recursively remove docker content.", + responses={202: AsyncOperationResponseSerializer}, + request_body=serializers.RecursiveManageSerializer, + ) + def create(self, request): + """ + Queues a task that creates a new RepositoryVersion by removing content units. + """ + remove_content_units = [] + serializer = serializers.RecursiveManageSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + repository = serializer.validated_data['repository'] + + if 'content_units' in request.data: + for url in request.data['content_units']: + content = NamedModelViewSet.get_resource(url, Content) + remove_content_units.append(content.pk) + + result = enqueue_with_reservation( + tasks.recursive_remove_content, [repository], + kwargs={ + 'repository_pk': repository.pk, + 'content_units': remove_content_units, + } + ) + return OperationPostponedResponse(result, request) diff --git a/pulp_docker/constants.py b/pulp_docker/constants.py new file mode 100644 index 00000000..5dcd6abd --- /dev/null +++ b/pulp_docker/constants.py @@ -0,0 +1,12 @@ +from types import SimpleNamespace + + +MEDIA_TYPE = SimpleNamespace( + MANIFEST_V1='application/vnd.docker.distribution.manifest.v1+json', + MANIFEST_V1_SIGNED='application/vnd.docker.distribution.manifest.v1+prettyjws', + MANIFEST_V2='application/vnd.docker.distribution.manifest.v2+json', + MANIFEST_LIST='application/vnd.docker.distribution.manifest.list.v2+json', + CONFIG_BLOB='application/vnd.docker.container.image.v1+json', + REGULAR_BLOB='application/vnd.docker.image.rootfs.diff.tar.gzip', + FOREIGN_BLOB='application/vnd.docker.image.rootfs.foreign.diff.tar.gzip', +) diff --git a/extensions_admin/pulp_docker/extensions/admin/__init__.py b/pulp_docker/tests/__init__.py similarity index 100% rename from extensions_admin/pulp_docker/extensions/admin/__init__.py rename to pulp_docker/tests/__init__.py diff --git a/pulp_docker/tests/functional/__init__.py b/pulp_docker/tests/functional/__init__.py new file mode 100644 index 00000000..45bf2ccc --- /dev/null +++ b/pulp_docker/tests/functional/__init__.py @@ -0,0 +1,2 @@ +# coding=utf-8 +"""Tests for docker plugin.""" diff --git a/pulp_docker/tests/functional/api/__init__.py b/pulp_docker/tests/functional/api/__init__.py new file mode 100644 index 00000000..8bd5b1fc --- /dev/null +++ b/pulp_docker/tests/functional/api/__init__.py @@ -0,0 +1,2 @@ +# coding=utf-8 +"""Tests that communicate with docker plugin via the v3 API.""" diff --git a/pulp_docker/tests/functional/api/test_crud_content_unit.py b/pulp_docker/tests/functional/api/test_crud_content_unit.py new file mode 100644 index 00000000..b4495d48 --- /dev/null +++ b/pulp_docker/tests/functional/api/test_crud_content_unit.py @@ -0,0 +1,105 @@ +# coding=utf-8 +"""Tests that CRUD docker content units.""" +import unittest + +from requests.exceptions import HTTPError + +from pulp_smash import api, config, utils +from pulp_smash.pulp3.constants import ARTIFACTS_PATH +from pulp_smash.pulp3.utils import delete_orphans + +from pulp_docker.tests.functional.constants import DOCKER_IMAGE_URL, DOCKER_CONTENT_PATH +from pulp_docker.tests.functional.utils import gen_docker_image_attrs, skip_if +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + + +# Read the instructions provided below for the steps needed to enable this test (see: FIXME's). +@unittest.skip( + "FIXME: plugin writer action required" + " docker plugin doesn't support push or uploads yet." +) +class ContentUnitTestCase(unittest.TestCase): + """CRUD content unit. + + This test targets the following issues: + + * `Pulp #2872 `_ + * `Pulp #3445 `_ + * `Pulp Smash #870 `_ + """ + + @classmethod + def setUpClass(cls): + """Create class-wide variable.""" + cls.cfg = config.get_config() + delete_orphans(cls.cfg) + cls.content_unit = {} + cls.client = api.Client(cls.cfg, api.json_handler) + files = {'file': utils.http_get(DOCKER_IMAGE_URL)} + cls.artifact = cls.client.post(ARTIFACTS_PATH, files=files) + + @classmethod + def tearDownClass(cls): + """Clean class-wide variable.""" + delete_orphans(cls.cfg) + + def test_01_create_content_unit(self): + """Create content unit.""" + attrs = gen_docker_image_attrs(self.artifact) + self.content_unit.update(self.client.post(DOCKER_CONTENT_PATH, attrs)) + for key, val in attrs.items(): + with self.subTest(key=key): + self.assertEqual(self.content_unit[key], val) + + @skip_if(bool, 'content_unit', False) + def test_02_read_content_unit(self): + """Read a content unit by its href.""" + content_unit = self.client.get(self.content_unit['pulp_href']) + for key, val in self.content_unit.items(): + with self.subTest(key=key): + self.assertEqual(content_unit[key], val) + + @skip_if(bool, 'content_unit', False) + def test_02_read_content_units(self): + """Read a content unit by its relative_path.""" + # FIXME: 'relative_path' is an attribute specific to the File plugin. It is only an + # example. You should replace this with some other field specific to your content type. + page = self.client.get(DOCKER_CONTENT_PATH, params={ + 'relative_path': self.content_unit['relative_path'] + }) + self.assertEqual(len(page['results']), 1) + for key, val in self.content_unit.items(): + with self.subTest(key=key): + self.assertEqual(page['results'][0][key], val) + + @skip_if(bool, 'content_unit', False) + def test_03_partially_update(self): + """Attempt to update a content unit using HTTP PATCH. + + This HTTP method is not supported and a HTTP exception is expected. + """ + attrs = gen_docker_image_attrs(self.artifact) + with self.assertRaises(HTTPError) as exc: + self.client.patch(self.content_unit['pulp_href'], attrs) + self.assertEqual(exc.exception.response.status_code, 405) + + @skip_if(bool, 'content_unit', False) + def test_03_fully_update(self): + """Attempt to update a content unit using HTTP PUT. + + This HTTP method is not supported and a HTTP exception is expected. + """ + attrs = gen_docker_image_attrs(self.artifact) + with self.assertRaises(HTTPError) as exc: + self.client.put(self.content_unit['pulp_href'], attrs) + self.assertEqual(exc.exception.response.status_code, 405) + + @skip_if(bool, 'content_unit', False) + def test_04_delete(self): + """Attempt to delete a content unit using HTTP DELETE. + + This HTTP method is not supported and a HTTP exception is expected. + """ + with self.assertRaises(HTTPError) as exc: + self.client.delete(self.content_unit['pulp_href']) + self.assertEqual(exc.exception.response.status_code, 405) diff --git a/pulp_docker/tests/functional/api/test_crud_distributions.py b/pulp_docker/tests/functional/api/test_crud_distributions.py new file mode 100644 index 00000000..029e3b3e --- /dev/null +++ b/pulp_docker/tests/functional/api/test_crud_distributions.py @@ -0,0 +1,212 @@ +# coding=utf-8 +"""Tests that CRUD distributions.""" +import unittest + +from itertools import permutations +from requests.exceptions import HTTPError + +from pulp_smash import api, config, selectors, utils +from pulp_smash.pulp3.utils import gen_distribution + +from pulp_docker.tests.functional.constants import DOCKER_DISTRIBUTION_PATH +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 +from pulp_docker.tests.functional.utils import skip_if + + +class CRUDDockerDistributionsTestCase(unittest.TestCase): + """CRUD distributions.""" + + @classmethod + def setUpClass(cls): + """Create class wide-variables.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.distribution = {} + + def test_01_create_distribution(self): + """Create a distribution.""" + body = gen_distribution() + response_dict = self.client.post( + DOCKER_DISTRIBUTION_PATH, body + ) + dist_task = self.client.get(response_dict['task']) + distribution_href = dist_task['created_resources'][0] + type(self).distribution = self.client.get(distribution_href) + for key, val in body.items(): + with self.subTest(key=key): + self.assertEqual(self.distribution[key], val) + + def test_02_create_same_name(self): + """Try to create a second distribution with an identical name. + + See: `Pulp Smash #1055 + `_. + """ + body = gen_distribution() + body['name'] = self.distribution['name'] + with self.assertRaises(HTTPError): + self.client.post(DOCKER_DISTRIBUTION_PATH, body) + + @skip_if(bool, 'distribution', False) + def test_02_read_distribution(self): + """Read a distribution by its pulp_href.""" + distribution = self.client.get(self.distribution['pulp_href']) + for key, val in self.distribution.items(): + with self.subTest(key=key): + self.assertEqual(distribution[key], val) + + @skip_if(bool, 'distribution', False) + def test_02_read_distribution_with_specific_fields(self): + """Read a distribution by its href providing specific field list. + + Permutate field list to ensure different combinations on result. + """ + if not selectors.bug_is_fixed(4599, self.cfg.pulp_version): + raise unittest.SkipTest('Issue 4599 is not resolved') + fields = ('base_path', 'name') + for field_pair in permutations(fields, 2): + # ex: field_pair = ('pulp_href', 'base_url) + with self.subTest(field_pair=field_pair): + distribution = self.client.get( + self.distribution['pulp_href'], + params={'fields': ','.join(field_pair)} + ) + self.assertEqual( + sorted(field_pair), sorted(distribution.keys()) + ) + + @skip_if(bool, 'distribution', False) + def test_02_read_distribution_without_specific_fields(self): + """Read a distribution by its href excluding specific fields.""" + if not selectors.bug_is_fixed(4599, self.cfg.pulp_version): + raise unittest.SkipTest('Issue 4599 is not resolved') + # requests doesn't allow the use of != in parameters. + url = '{}?exclude_fields=base_path,name'.format( + self.distribution['pulp_href'] + ) + distribution = self.client.get(url) + response_fields = distribution.keys() + self.assertNotIn('base_path', response_fields) + self.assertNotIn('name', response_fields) + + @skip_if(bool, 'distribution', False) + def test_02_read_distributions(self): + """Read a distribution using query parameters. + + See: `Pulp #3082 `_ + """ + unique_params = ( + {'name': self.distribution['name']}, + {'base_path': self.distribution['base_path']} + ) + for params in unique_params: + with self.subTest(params=params): + page = self.client.get(DOCKER_DISTRIBUTION_PATH, params=params) + self.assertEqual(len(page['results']), 1) + for key, val in self.distribution.items(): + with self.subTest(key=key): + self.assertEqual(page['results'][0][key], val) + + @skip_if(bool, 'distribution', False) + def test_03_partially_update(self): + """Update a distribution using HTTP PATCH.""" + body = gen_distribution() + self.client.patch(self.distribution['pulp_href'], body) + type(self).distribution = self.client.get(self.distribution['pulp_href']) + for key, val in body.items(): + with self.subTest(key=key): + self.assertEqual(self.distribution[key], val) + + @skip_if(bool, 'distribution', False) + def test_04_fully_update(self): + """Update a distribution using HTTP PUT.""" + body = gen_distribution() + self.client.put(self.distribution['pulp_href'], body) + type(self).distribution = self.client.get(self.distribution['pulp_href']) + for key, val in body.items(): + with self.subTest(key=key): + self.assertEqual(self.distribution[key], val) + + @skip_if(bool, 'distribution', False) + def test_05_delete(self): + """Delete a distribution.""" + self.client.delete(self.distribution['pulp_href']) + with self.assertRaises(HTTPError): + self.client.get(self.distribution['pulp_href']) + + def test_negative_create_distribution_with_invalid_parameter(self): + """Attempt to create distribution passing invalid parameter. + + Assert response returns an error 400 including ["Unexpected field"]. + """ + response = api.Client(self.cfg, api.echo_handler).post( + DOCKER_DISTRIBUTION_PATH, gen_distribution(foo='bar') + ) + assert response.status_code == 400 + assert response.json()['foo'] == ['Unexpected field'] + + +class DistributionBasePathTestCase(unittest.TestCase): + """Test possible values for ``base_path`` on a distribution. + + This test targets the following issues: + + * `Pulp #2987 `_ + * `Pulp #3412 `_ + * `Pulp Smash #906 `_ + * `Pulp Smash #956 `_ + """ + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + body = gen_distribution() + body['base_path'] = body['base_path'].replace('-', '/') + response_dict = cls.client.post(DOCKER_DISTRIBUTION_PATH, body) + dist_task = cls.client.get(response_dict['task']) + distribution_href = dist_task['created_resources'][0] + cls.distribution = cls.client.get(distribution_href) + + @classmethod + def tearDownClass(cls): + """Clean up resources.""" + cls.client.delete(cls.distribution['pulp_href']) + + def test_spaces(self): + """Test that spaces can not be part of ``base_path``.""" + self.try_create_distribution(base_path=utils.uuid4().replace('-', ' ')) + self.try_update_distribution(base_path=utils.uuid4().replace('-', ' ')) + + def test_begin_slash(self): + """Test that slash cannot be in the begin of ``base_path``.""" + self.try_create_distribution(base_path='/' + utils.uuid4()) + self.try_update_distribution(base_path='/' + utils.uuid4()) + + def test_end_slash(self): + """Test that slash cannot be in the end of ``base_path``.""" + self.try_create_distribution(base_path=utils.uuid4() + '/') + self.try_update_distribution(base_path=utils.uuid4() + '/') + + def test_unique_base_path(self): + """Test that ``base_path`` can not be duplicated.""" + self.try_create_distribution(base_path=self.distribution['base_path']) + + def try_create_distribution(self, **kwargs): + """Unsuccessfully create a distribution. + + Merge the given kwargs into the body of the request. + """ + body = gen_distribution() + body.update(kwargs) + with self.assertRaises(HTTPError): + self.client.post(DOCKER_DISTRIBUTION_PATH, body) + + def try_update_distribution(self, **kwargs): + """Unsuccessfully update a distribution with HTTP PATCH. + + Use the given kwargs as the body of the request. + """ + with self.assertRaises(HTTPError): + self.client.patch(self.distribution['pulp_href'], kwargs) diff --git a/pulp_docker/tests/functional/api/test_crud_remotes.py b/pulp_docker/tests/functional/api/test_crud_remotes.py new file mode 100644 index 00000000..de02a4ff --- /dev/null +++ b/pulp_docker/tests/functional/api/test_crud_remotes.py @@ -0,0 +1,137 @@ +# coding=utf-8 +"""Tests that CRUD docker remotes.""" +from random import choice +import unittest + +from requests.exceptions import HTTPError + +from pulp_smash import api, config, utils +from pulp_smash.pulp3.constants import ( + IMMEDIATE_DOWNLOAD_POLICIES, + ON_DEMAND_DOWNLOAD_POLICIES, +) + +from pulp_docker.tests.functional.constants import DOCKER_REMOTE_PATH +from pulp_docker.tests.functional.utils import skip_if, gen_docker_remote +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + + +class CRUDRemotesTestCase(unittest.TestCase): + """CRUD remotes.""" + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.remote = {} + + def test_01_create_remote(self): + """Create a remote.""" + body = _gen_verbose_remote() + type(self).remote = self.client.post(DOCKER_REMOTE_PATH, body) + for key in ('username', 'password'): + del body[key] + for key, val in body.items(): + with self.subTest(key=key): + self.assertEqual(self.remote[key], val) + + @skip_if(bool, 'remote', False) + def test_02_create_same_name(self): + """Try to create a second remote with an identical name. + + See: `Pulp Smash #1055 + `_. + """ + body = gen_docker_remote() + body['name'] = self.remote['name'] + with self.assertRaises(HTTPError): + self.client.post(DOCKER_REMOTE_PATH, body) + + @skip_if(bool, 'remote', False) + def test_02_read_remote(self): + """Read a remote by its href.""" + remote = self.client.get(self.remote['pulp_href']) + for key, val in self.remote.items(): + with self.subTest(key=key): + self.assertEqual(remote[key], val) + + @skip_if(bool, 'remote', False) + def test_02_read_remotes(self): + """Read a remote by its name.""" + page = self.client.get(DOCKER_REMOTE_PATH, params={ + 'name': self.remote['name'] + }) + self.assertEqual(len(page['results']), 1) + for key, val in self.remote.items(): + with self.subTest(key=key): + self.assertEqual(page['results'][0][key], val) + + @skip_if(bool, 'remote', False) + def test_03_partially_update(self): + """Update a remote using HTTP PATCH.""" + body = _gen_verbose_remote() + self.client.patch(self.remote['pulp_href'], body) + for key in ('username', 'password'): + del body[key] + type(self).remote = self.client.get(self.remote['pulp_href']) + for key, val in body.items(): + with self.subTest(key=key): + self.assertEqual(self.remote[key], val) + + @skip_if(bool, 'remote', False) + def test_04_fully_update(self): + """Update a remote using HTTP PUT.""" + body = _gen_verbose_remote() + self.client.put(self.remote['pulp_href'], body) + for key in ('username', 'password'): + del body[key] + type(self).remote = self.client.get(self.remote['pulp_href']) + for key, val in body.items(): + with self.subTest(key=key): + self.assertEqual(self.remote[key], val) + + @skip_if(bool, 'remote', False) + def test_05_delete(self): + """Delete a remote.""" + self.client.delete(self.remote['pulp_href']) + with self.assertRaises(HTTPError): + self.client.get(self.remote['pulp_href']) + + +class CreateRemoteNoURLTestCase(unittest.TestCase): + """Verify whether is possible to create a remote without a URL.""" + + def test_all(self): + """Verify whether is possible to create a remote without a URL. + + This test targets the following issues: + + * `Pulp #3395 `_ + * `Pulp Smash #984 `_ + """ + body = gen_docker_remote() + del body['url'] + with self.assertRaises(HTTPError): + api.Client(config.get_config()).post(DOCKER_REMOTE_PATH, body) + + +def _gen_verbose_remote(): + """Return a semi-random dict for use in defining a remote. + + For most tests, it's desirable to create remotes with as few attributes + as possible, so that the tests can specifically target and attempt to break + specific features. This module specifically targets remotes, so it makes + sense to provide as many attributes as possible. + + Note that 'username' and 'password' are write-only attributes. + """ + attrs = gen_docker_remote() + attrs.update({ + 'password': utils.uuid4(), + 'username': utils.uuid4(), + 'policy': choice( + IMMEDIATE_DOWNLOAD_POLICIES + ON_DEMAND_DOWNLOAD_POLICIES + ), + }) + return attrs diff --git a/pulp_docker/tests/functional/api/test_pull_content.py b/pulp_docker/tests/functional/api/test_pull_content.py new file mode 100644 index 00000000..9359bd3b --- /dev/null +++ b/pulp_docker/tests/functional/api/test_pull_content.py @@ -0,0 +1,462 @@ +# coding=utf-8 +"""Tests that verify that images served by Pulp can be pulled.""" +import contextlib +import unittest +from urllib.parse import urljoin + +from pulp_smash import api, cli, config, exceptions +from pulp_smash.pulp3.constants import ARTIFACTS_PATH, REPO_PATH +from pulp_smash.pulp3.utils import ( + delete_orphans, + get_content, + gen_distribution, + gen_repo, + sync, +) + +from pulp_docker.tests.functional.utils import ( + gen_docker_remote, + get_docker_hub_remote_blobsums +) + +from pulp_docker.tests.functional.constants import ( + DOCKER_CONTENT_NAME, + DOCKER_DISTRIBUTION_PATH, + DOCKER_REMOTE_PATH, + DOCKER_UPSTREAM_NAME, + DOCKER_UPSTREAM_TAG, +) +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + + +class PullContentTestCase(unittest.TestCase): + """Verify whether images served by Pulp can be pulled.""" + + @classmethod + def setUpClass(cls): + """Create class-wide variables. + + 1. Create a repository. + 2. Create a remote pointing to external registry. + 3. Sync the repository using the remote and re-read the repo data. + 4. Create a docker distribution to serve the repository + 5. Create another docker distribution to the serve the repository version + + This tests targets the following issue: + + * `Pulp #4460 `_ + """ + cls.cfg = config.get_config() + + token_auth = cls.cfg.hosts[0].roles['token auth'] + client = cli.Client(cls.cfg) + client.run('openssl ecparam -genkey -name prime256v1 -noout -out {}' + .format(token_auth['private key']).split()) + client.run('openssl ec -in {} -pubout -out {}'.format( + token_auth['private key'], token_auth['public key']).split()) + + cls.client = api.Client(cls.cfg, api.page_handler) + cls.teardown_cleanups = [] + + with contextlib.ExitStack() as stack: + # ensure tearDownClass runs if an error occurs here + stack.callback(cls.tearDownClass) + + # Step 1 + _repo = cls.client.post(REPO_PATH, gen_repo()) + cls.teardown_cleanups.append((cls.client.delete, _repo['pulp_href'])) + + # Step 2 + cls.remote = cls.client.post( + DOCKER_REMOTE_PATH, gen_docker_remote() + ) + cls.teardown_cleanups.append( + (cls.client.delete, cls.remote['pulp_href']) + ) + + # Step 3 + sync(cls.cfg, cls.remote, _repo) + cls.repo = cls.client.get(_repo['pulp_href']) + + # Step 4. + response_dict = cls.client.using_handler(api.task_handler).post( + DOCKER_DISTRIBUTION_PATH, + gen_distribution(repository=cls.repo['pulp_href']) + ) + distribution_href = response_dict['pulp_href'] + cls.distribution_with_repo = cls.client.get(distribution_href) + cls.teardown_cleanups.append( + (cls.client.delete, cls.distribution_with_repo['pulp_href']) + ) + + # Step 5. + response_dict = cls.client.using_handler(api.task_handler).post( + DOCKER_DISTRIBUTION_PATH, + gen_distribution(repository_version=cls.repo['latest_version_href']) + ) + distribution_href = response_dict['pulp_href'] + cls.distribution_with_repo_version = cls.client.get(distribution_href) + cls.teardown_cleanups.append( + (cls.client.delete, cls.distribution_with_repo_version['pulp_href']) + ) + + # remove callback if everything goes well + stack.pop_all() + + @classmethod + def tearDownClass(cls): + """Clean class-wide variable.""" + for cleanup_function, args in reversed(cls.teardown_cleanups): + cleanup_function(args) + + def test_api_returns_same_checksum(self): + """Verify that pulp serves image with the same checksum of remote. + + 1. Call pulp repository API and get the content_summary for repo. + 2. Call dockerhub API and get blobsums for synced image. + 3. Compare the checksums. + """ + # Get local checksums for content synced from remote registy + checksums = [ + content['digest'] for content + in get_content(self.repo)[DOCKER_CONTENT_NAME] + ] + + # Assert that at least one layer is synced from remote:latest + # and the checksum matched with remote + self.assertTrue( + any( + [ + result['blobSum'] in checksums + for result in get_docker_hub_remote_blobsums() + ] + ), + 'Cannot find a matching layer on remote registry.' + ) + + def test_pull_image_from_repository(self): + """Verify that a client can pull the image from Pulp. + + 1. Using the RegistryClient pull the image from Pulp. + 2. Pull the same image from remote registry. + 3. Verify both images has the same checksum. + 4. Ensure image is deleted after the test. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution_with_repo['base_path'] + ) + + registry.pull(local_url) + self.teardown_cleanups.append((registry.rmi, local_url)) + local_image = registry.inspect(local_url) + + registry.pull(DOCKER_UPSTREAM_NAME) + remote_image = registry.inspect(DOCKER_UPSTREAM_NAME) + + self.assertEqual( + local_image[0]['Id'], + remote_image[0]['Id'] + ) + registry.rmi(DOCKER_UPSTREAM_NAME) + + def test_pull_image_from_repository_version(self): + """Verify that a client can pull the image from Pulp. + + 1. Using the RegistryClient pull the image from Pulp. + 2. Pull the same image from remote registry. + 3. Verify both images has the same checksum. + 4. Ensure image is deleted after the test. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution_with_repo_version['base_path'] + ) + + registry.pull(local_url) + self.teardown_cleanups.append((registry.rmi, local_url)) + local_image = registry.inspect(local_url) + + registry.pull(DOCKER_UPSTREAM_NAME) + remote_image = registry.inspect(DOCKER_UPSTREAM_NAME) + + self.assertEqual( + local_image[0]['Id'], + remote_image[0]['Id'] + ) + registry.rmi(DOCKER_UPSTREAM_NAME) + + def test_pull_image_with_tag(self): + """Verify that a client can pull the image from Pulp with a tag. + + 1. Using the RegistryClient pull the image from Pulp specifying a tag. + 2. Pull the same image and same tag from remote registry. + 3. Verify both images has the same checksum. + 4. Ensure image is deleted after the test. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution_with_repo['base_path'] + ) + DOCKER_UPSTREAM_TAG + + registry.pull(local_url) + self.teardown_cleanups.append((registry.rmi, local_url)) + local_image = registry.inspect(local_url) + + registry.pull(DOCKER_UPSTREAM_NAME + DOCKER_UPSTREAM_TAG) + self.teardown_cleanups.append( + (registry.rmi, DOCKER_UPSTREAM_NAME + DOCKER_UPSTREAM_TAG) + ) + remote_image = registry.inspect( + DOCKER_UPSTREAM_NAME + DOCKER_UPSTREAM_TAG + ) + + self.assertEqual( + local_image[0]['Id'], + remote_image[0]['Id'] + ) + + def test_pull_nonexistent_image(self): + """Verify that a client cannot pull nonexistent image from Pulp. + + 1. Using the RegistryClient try to pull nonexistent image from Pulp. + 2. Assert that error is occurred and nothing has been pulled. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + "inexistentimagename" + ) + with self.assertRaises(exceptions.CalledProcessError): + registry.pull(local_url) + + +class PullOnDemandContentTestCase(unittest.TestCase): + """Verify whether on-demand served images by Pulp can be pulled.""" + + @classmethod + def setUpClass(cls): + """Create class-wide variables and delete orphans. + + 1. Create a repository. + 2. Create a remote pointing to external registry with policy=on_demand. + 3. Sync the repository using the remote and re-read the repo data. + 4. Create a docker distribution to serve the repository + 5. Create another docker distribution to the serve the repository version + + This tests targets the following issue: + + * `Pulp #4460 `_ + """ + cls.cfg = config.get_config() + + token_auth = cls.cfg.hosts[0].roles['token auth'] + client = cli.Client(cls.cfg) + client.run('openssl ecparam -genkey -name prime256v1 -noout -out {}' + .format(token_auth['private key']).split()) + client.run('openssl ec -in {} -pubout -out {}'.format( + token_auth['private key'], token_auth['public key']).split()) + + cls.client = api.Client(cls.cfg, api.page_handler) + + cls.teardown_cleanups = [] + + delete_orphans(cls.cfg) + + with contextlib.ExitStack() as stack: + # ensure tearDownClass runs if an error occurs here + stack.callback(cls.tearDownClass) + + # Step 1 + _repo = cls.client.post(REPO_PATH, gen_repo()) + cls.teardown_cleanups.append((cls.client.delete, _repo['pulp_href'])) + + # Step 2 + cls.remote = cls.client.post( + DOCKER_REMOTE_PATH, gen_docker_remote(policy='on_demand') + ) + cls.teardown_cleanups.append( + (cls.client.delete, cls.remote['pulp_href']) + ) + + # Step 3 + sync(cls.cfg, cls.remote, _repo) + cls.repo = cls.client.get(_repo['pulp_href']) + cls.artifact_count = len(cls.client.get(ARTIFACTS_PATH)) + + # Step 4. + response_dict = cls.client.using_handler(api.task_handler).post( + DOCKER_DISTRIBUTION_PATH, + gen_distribution(repository=cls.repo['pulp_href']) + ) + distribution_href = response_dict['pulp_href'] + cls.distribution_with_repo = cls.client.get(distribution_href) + cls.teardown_cleanups.append( + (cls.client.delete, cls.distribution_with_repo['pulp_href']) + ) + + # Step 5. + response_dict = cls.client.using_handler(api.task_handler).post( + DOCKER_DISTRIBUTION_PATH, + gen_distribution(repository_version=cls.repo['latest_version_href']) + ) + distribution_href = response_dict['pulp_href'] + cls.distribution_with_repo_version = cls.client.get(distribution_href) + cls.teardown_cleanups.append( + (cls.client.delete, cls.distribution_with_repo_version['pulp_href']) + ) + + # remove callback if everything goes well + stack.pop_all() + + @classmethod + def tearDownClass(cls): + """Clean class-wide variable.""" + for cleanup_function, args in reversed(cls.teardown_cleanups): + cleanup_function(args) + + def test_api_returns_same_checksum(self): + """Verify that pulp serves image with the same checksum of remote. + + 1. Call pulp repository API and get the content_summary for repo. + 2. Call dockerhub API and get blobsums for synced image. + 3. Compare the checksums. + """ + # Get local checksums for content synced from remote registy + checksums = [ + content['digest'] for content + in get_content(self.repo)[DOCKER_CONTENT_NAME] + ] + + # Assert that at least one layer is synced from remote:latest + # and the checksum matched with remote + self.assertTrue( + any( + [ + result['blobSum'] in checksums + for result in get_docker_hub_remote_blobsums() + ] + ), + 'Cannot find a matching layer on remote registry.' + ) + + def test_pull_image_from_repository(self): + """Verify that a client can pull the image from Pulp (on-demand). + + 1. Using the RegistryClient pull the image from Pulp. + 2. Pull the same image from remote registry. + 3. Verify both images has the same checksum. + 4. Verify that the number of artifacts in Pulp has increased. + 5. Ensure image is deleted after the test. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution_with_repo['base_path'] + ) + + registry.pull(local_url) + self.teardown_cleanups.append((registry.rmi, local_url)) + local_image = registry.inspect(local_url) + + registry.pull(DOCKER_UPSTREAM_NAME) + remote_image = registry.inspect(DOCKER_UPSTREAM_NAME) + + self.assertEqual( + local_image[0]['Id'], + remote_image[0]['Id'] + ) + + new_artifact_count = len(self.client.get(ARTIFACTS_PATH)) + self.assertGreater(new_artifact_count, self.artifact_count) + + registry.rmi(DOCKER_UPSTREAM_NAME) + + def test_pull_image_from_repository_version(self): + """Verify that a client can pull the image from Pulp (on-demand). + + 1. Using the RegistryClient pull the image from Pulp. + 2. Pull the same image from remote registry. + 3. Verify both images has the same checksum. + 4. Ensure image is deleted after the test. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution_with_repo_version['base_path'] + ) + + registry.pull(local_url) + self.teardown_cleanups.append((registry.rmi, local_url)) + local_image = registry.inspect(local_url) + + registry.pull(DOCKER_UPSTREAM_NAME) + remote_image = registry.inspect(DOCKER_UPSTREAM_NAME) + + self.assertEqual( + local_image[0]['Id'], + remote_image[0]['Id'] + ) + registry.rmi(DOCKER_UPSTREAM_NAME) + + def test_pull_image_with_tag(self): + """Verify that a client can pull the image from Pulp with a tag (on-demand). + + 1. Using the RegistryClient pull the image from Pulp specifying a tag. + 2. Pull the same image and same tag from remote registry. + 3. Verify both images has the same checksum. + 4. Ensure image is deleted after the test. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported( + unittest.SkipTest, 'Test requires podman/docker' + ) + + local_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution_with_repo['base_path'] + ) + DOCKER_UPSTREAM_TAG + + registry.pull(local_url) + self.teardown_cleanups.append((registry.rmi, local_url)) + local_image = registry.inspect(local_url) + + registry.pull(DOCKER_UPSTREAM_NAME + DOCKER_UPSTREAM_TAG) + self.teardown_cleanups.append( + (registry.rmi, DOCKER_UPSTREAM_NAME + DOCKER_UPSTREAM_TAG) + ) + remote_image = registry.inspect( + DOCKER_UPSTREAM_NAME + DOCKER_UPSTREAM_TAG + ) + + self.assertEqual( + local_image[0]['Id'], + remote_image[0]['Id'] + ) diff --git a/pulp_docker/tests/functional/api/test_recursive_add.py b/pulp_docker/tests/functional/api/test_recursive_add.py new file mode 100644 index 00000000..f3b29b18 --- /dev/null +++ b/pulp_docker/tests/functional/api/test_recursive_add.py @@ -0,0 +1,672 @@ +# coding=utf-8 +"""Tests that recursively add docker content to repositories.""" +import unittest + +from pulp_smash import api, config +from pulp_smash.pulp3.constants import REPO_PATH +from pulp_smash.pulp3.utils import gen_repo, sync +from requests.exceptions import HTTPError + +from pulp_docker.tests.functional.constants import ( + DOCKER_MANIFEST_COPY_PATH, + DOCKER_TAG_PATH, + DOCKER_TAG_COPY_PATH, + DOCKER_TAGGING_PATH, + DOCKER_REMOTE_PATH, + DOCKER_RECURSIVE_ADD_PATH, + DOCKERHUB_PULP_FIXTURE_1, +) +from pulp_docker.tests.functional.utils import gen_docker_remote +from pulp_docker.constants import MEDIA_TYPE + + +class TestManifestCopy(unittest.TestCase): + """ + Test recursive copy of Manifests into a repository. + + This test targets the follow feature: + https://pulp.plan.io/issues/3403 + """ + + @classmethod + def setUpClass(cls): + """Sync pulp/test-fixture-1 so we can copy content from it.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.from_repo = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + sync(cls.cfg, cls.remote, cls.from_repo) + latest_version = cls.client.get(cls.from_repo['pulp_href'])['latest_version_href'] + cls.latest_from_version = "repository_version={version}".format(version=latest_version) + + def setUp(self): + """Create an empty repository to copy into.""" + self.to_repo = self.client.post(REPO_PATH, gen_repo()) + self.addCleanup(self.client.delete, self.to_repo['pulp_href']) + + @classmethod + def tearDownClass(cls): + """Delete things made in setUpClass. addCleanup feature does not work with setupClass.""" + cls.client.delete(cls.from_repo['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + + def test_missing_repository_argument(self): + """Ensure source_repository or source_repository_version is required.""" + with self.assertRaises(HTTPError) as context: + self.client.post(DOCKER_MANIFEST_COPY_PATH) + self.assertEqual(context.exception.response.status_code, 400) + + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + {'source_repository': self.from_repo['pulp_href']} + ) + self.assertEqual(context.exception.response.status_code, 400) + + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + {'source_repository_version': self.from_repo['latest_version_href']} + ) + self.assertEqual(context.exception.response.status_code, 400) + + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'destination_repository': self.to_repo['pulp_href']} + ) + self.assertEqual(context.exception.response.status_code, 400) + + def test_empty_source_repository(self): + """Ensure exception is raised when source_repository does not have latest version.""" + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + # to_repo has no versions, use it as source + 'source_repository': self.to_repo['pulp_href'], + 'destination_repository': self.from_repo['pulp_href'], + } + ) + self.assertEqual(context.exception.response.status_code, 400) + + def test_source_repository_and_source_version(self): + """Passing source_repository_version and repository returns a 400.""" + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'source_repository_version': self.from_repo['latest_version_href'], + 'destination_repository': self.to_repo['pulp_href'] + } + ) + self.assertEqual(context.exception.response.status_code, 400) + + def test_copy_all_manifests(self): + """Passing only source and destination repositories copies all manifests.""" + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest_from_repo_href = self.client.get(self.from_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + from_repo_content = self.client.get(latest_from_repo_href)['content_summary']['present'] + for docker_type in ['docker.manifest', 'docker.blob']: + self.assertEqual( + to_repo_content[docker_type]['count'], + from_repo_content[docker_type]['count'], + msg=docker_type, + ) + self.assertFalse('docker.tag' in to_repo_content) + + def test_copy_all_manifests_from_version(self): + """Passing only source version and destination repositories copies all manifests.""" + latest_from_repo_href = self.client.get(self.from_repo['pulp_href'])['latest_version_href'] + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository_version': latest_from_repo_href, + 'destination_repository': self.to_repo['pulp_href'] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + from_repo_content = self.client.get(latest_from_repo_href)['content_summary']['present'] + for docker_type in ['docker.manifest', 'docker.blob']: + self.assertEqual( + to_repo_content[docker_type]['count'], + from_repo_content[docker_type]['count'] + ) + self.assertFalse('docker.tag' in to_repo_content) + + def test_copy_manifest_by_digest(self): + """Specify a single manifest by digest to copy.""" + manifest_a_href = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + manifest_a_digest = self.client.get(manifest_a_href)['digest'] + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'digests': [manifest_a_digest] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + self.assertFalse('docker.tag' in to_repo_content) + self.assertEqual(to_repo_content['docker.manifest']['count'], 1) + # manifest_a has 2 blobs + self.assertEqual(to_repo_content['docker.blob']['count'], 2) + + def test_copy_manifest_by_digest_and_media_type(self): + """Specify a single manifest by digest to copy.""" + manifest_a_href = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + manifest_a_digest = self.client.get(manifest_a_href)['digest'] + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'digests': [manifest_a_digest], + 'media_types': [MEDIA_TYPE.MANIFEST_V2] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + self.assertFalse('docker.tag' in to_repo_content) + self.assertEqual(to_repo_content['docker.manifest']['count'], 1) + # manifest_a has 2 blobs + self.assertEqual(to_repo_content['docker.blob']['count'], 2) + + def test_copy_all_manifest_lists_by_media_type(self): + """Specify the media_type, to copy all manifest lists.""" + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'media_types': [MEDIA_TYPE.MANIFEST_LIST] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + self.assertFalse('docker.tag' in to_repo_content) + # Fixture has 4 manifest lists, which combined reference 5 manifests + self.assertEqual(to_repo_content['docker.manifest']['count'], 9) + # each manifest (non-list) has 2 blobs + self.assertEqual(to_repo_content['docker.blob']['count'], 10) + + def test_copy_all_manifests_by_media_type(self): + """Specify the media_type, to copy all manifest lists.""" + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'media_types': [MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V2] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + self.assertFalse('docker.tag' in to_repo_content) + # Fixture has 5 manifests that aren't manifest lists + self.assertEqual(to_repo_content['docker.manifest']['count'], 5) + # each manifest (non-list) has 2 blobs + self.assertEqual(to_repo_content['docker.blob']['count'], 10) + + def test_fail_to_copy_invalid_manifest_media_type(self): + """Specify the media_type, to copy all manifest lists.""" + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'media_types': ['wrongwrongwrong'] + } + ) + self.assertEqual(context.exception.response.status_code, 400) + + def test_copy_by_digest_with_incorrect_media_type(self): + """Ensure invalid media type will raise a 400.""" + ml_i_href = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + ml_i_digest = self.client.get(ml_i_href)['digest'] + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'digests': [ml_i_digest], + 'media_types': [MEDIA_TYPE.MANIFEST_V2] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + # No content added + for docker_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertFalse(docker_type in to_repo_content, msg=docker_type) + + def test_copy_multiple_manifests_by_digest(self): + """Specify digests to copy.""" + ml_i_href = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + ml_i_digest = self.client.get(ml_i_href)['digest'] + ml_ii_href = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_ii&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + ml_ii_digest = self.client.get(ml_ii_href)['digest'] + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'digests': [ml_i_digest, ml_ii_digest] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + self.assertFalse('docker.tag' in to_repo_content) + # each manifest list is a manifest and references 2 other manifests + self.assertEqual(to_repo_content['docker.manifest']['count'], 6) + # each referenced manifest has 2 blobs + self.assertEqual(to_repo_content['docker.blob']['count'], 8) + + def test_copy_manifests_by_digest_empty_list(self): + """Passing an empty list copies no manifests.""" + self.client.post( + DOCKER_MANIFEST_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'digests': [] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + # A new version was created + self.assertNotEqual(latest_to_repo_href, self.to_repo['latest_version_href']) + + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + # No content added + for docker_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertFalse(docker_type in to_repo_content) + + +class TestTagCopy(unittest.TestCase): + """Test recursive copy of tags content to a repository.""" + + @classmethod + def setUpClass(cls): + """Sync pulp/test-fixture-1 so we can copy content from it.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.from_repo = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + sync(cls.cfg, cls.remote, cls.from_repo) + latest_version = cls.client.get(cls.from_repo['pulp_href'])['latest_version_href'] + cls.latest_from_version = "repository_version={version}".format(version=latest_version) + + def setUp(self): + """Create an empty repository to copy into.""" + self.to_repo = self.client.post(REPO_PATH, gen_repo()) + self.addCleanup(self.client.delete, self.to_repo['pulp_href']) + + @classmethod + def tearDownClass(cls): + """Delete things made in setUpClass. addCleanup feature does not work with setupClass.""" + cls.client.delete(cls.from_repo['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + + def test_missing_repository_argument(self): + """Ensure source_repository or source_repository_version is required.""" + with self.assertRaises(HTTPError): + self.client.post(DOCKER_RECURSIVE_ADD_PATH) + + with self.assertRaises(HTTPError): + self.client.post( + DOCKER_TAG_COPY_PATH, + {'source_repository': self.from_repo['pulp_href']} + ) + + with self.assertRaises(HTTPError): + self.client.post( + DOCKER_TAG_COPY_PATH, + {'source_repository_version': self.from_repo['latest_version_href']} + ) + + with self.assertRaises(HTTPError): + self.client.post( + DOCKER_TAG_COPY_PATH, + {'destination_repository': self.to_repo['pulp_href']} + ) + + def test_empty_source_repository(self): + """Ensure exception is raised when source_repository does not have latest version.""" + with self.assertRaises(HTTPError): + self.client.post( + DOCKER_TAG_COPY_PATH, + { + # to_repo has no versions, use it as source + 'source_repository': self.to_repo['pulp_href'], + 'destination_repository': self.from_repo['pulp_href'], + } + ) + + def test_source_repository_and_source_version(self): + """Passing both source_repository_version and source_repository returns a 400.""" + with self.assertRaises(HTTPError) as context: + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'source_repository_version': self.from_repo['latest_version_href'], + 'destination_repository': self.to_repo['pulp_href'] + } + ) + self.assertEqual(context.exception.response.status_code, 400) + + def test_copy_all_tags(self): + """Passing only source and destination repositories copies all tags.""" + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest_from_repo_href = self.client.get(self.from_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + from_repo_content = self.client.get(latest_from_repo_href)['content_summary']['present'] + for docker_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertEqual( + to_repo_content[docker_type]['count'], + from_repo_content[docker_type]['count'], + msg=docker_type, + ) + + def test_copy_all_tags_from_version(self): + """Passing only source version and destination repositories copies all tags.""" + latest_from_repo_href = self.client.get(self.from_repo['pulp_href'])['latest_version_href'] + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository_version': latest_from_repo_href, + 'destination_repository': self.to_repo['pulp_href'] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + from_repo_content = self.client.get(latest_from_repo_href)['content_summary']['present'] + for docker_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertEqual( + to_repo_content[docker_type]['count'], + from_repo_content[docker_type]['count'], + msg=docker_type, + ) + + def test_copy_tags_by_name(self): + """Copy tags in destination repo that match name.""" + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'names': ['ml_i', 'manifest_c'] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + self.assertEqual(to_repo_content['docker.tag']['count'], 2) + # ml_i has 1 manifest list, 2 manifests, manifest_c has 1 manifest + self.assertEqual(to_repo_content['docker.manifest']['count'], 4) + # each manifest (not manifest list) has 2 blobs + self.assertEqual(to_repo_content['docker.blob']['count'], 6) + + def test_copy_tags_by_name_empty_list(self): + """Passing an empty list of names copies nothing.""" + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'], + 'names': [] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + # A new version was created + self.assertNotEqual(latest_to_repo_href, self.to_repo['latest_version_href']) + + to_repo_content = self.client.get(latest_to_repo_href)['content_summary']['present'] + # No content added + for docker_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertFalse(docker_type in to_repo_content) + + def test_copy_tags_with_conflicting_names(self): + """If tag names are already present in a repository, the conflicting tags are removed.""" + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'] + } + ) + # Same call + self.client.post( + DOCKER_TAG_COPY_PATH, + { + 'source_repository': self.from_repo['pulp_href'], + 'destination_repository': self.to_repo['pulp_href'] + } + ) + latest_to_repo_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest_from_repo_href = self.client.get(self.from_repo['pulp_href'])['latest_version_href'] + to_repo_content = self.client.get(latest_to_repo_href)['content_summary'] + from_repo_content = self.client.get(latest_from_repo_href)['content_summary'] + for docker_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertEqual( + to_repo_content['present'][docker_type]['count'], + from_repo_content['present'][docker_type]['count'] + ) + + self.assertEqual( + to_repo_content['added']['docker.tag']['count'], + from_repo_content['present']['docker.tag']['count'] + ) + self.assertEqual( + to_repo_content['removed']['docker.tag']['count'], + from_repo_content['present']['docker.tag']['count'] + ) + + +class TestRecursiveAdd(unittest.TestCase): + """Test recursively adding docker content to a repository.""" + + @classmethod + def setUpClass(cls): + """Sync pulp/test-fixture-1 so we can copy content from it.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.from_repo = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + sync(cls.cfg, cls.remote, cls.from_repo) + latest_version = cls.client.get(cls.from_repo['pulp_href'])['latest_version_href'] + cls.latest_from_version = "repository_version={version}".format(version=latest_version) + + def setUp(self): + """Create an empty repository to copy into.""" + self.to_repo = self.client.post(REPO_PATH, gen_repo()) + self.addCleanup(self.client.delete, self.to_repo['pulp_href']) + + @classmethod + def tearDownClass(cls): + """Delete things made in setUpClass. addCleanup feature does not work with setupClass.""" + cls.client.delete(cls.from_repo['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + + def test_missing_repository_argument(self): + """Ensure Repository argument is required.""" + with self.assertRaises(HTTPError): + self.client.post(DOCKER_RECURSIVE_ADD_PATH) + + def test_repository_only(self): + """Passing only a repository creates a new version.""" + self.client.post(DOCKER_RECURSIVE_ADD_PATH, {'repository': self.to_repo['pulp_href']}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + self.assertNotEqual(latest_version_href, self.to_repo['latest_version_href']) + + def test_manifest_recursion(self): + """Add a manifest and its related blobs.""" + manifest_a = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + # No tags added + self.assertFalse('docker.manifest-tag' in latest['content_summary']['added']) + + # manifest a has 2 blobs + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 2) + + def test_manifest_list_recursion(self): + """Add a Manifest List, related manifests, and related blobs.""" + ml_i = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + # No tags added + self.assertFalse('docker.tag' in latest['content_summary']['added']) + # 1 manifest list 2 manifests + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 3) + + def test_tagged_manifest_list_recursion(self): + """Add a tagged manifest list, and its related manifests and blobs.""" + ml_i_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i_tag]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 1) + # 1 manifest list 2 manifests + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 3) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 4) + + def test_tagged_manifest_recursion(self): + """Add a tagged manifest and its related blobs.""" + manifest_a_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a_tag]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 2) + + def test_tag_replacement(self): + """Add a tagged manifest to a repo with a tag of that name already in place.""" + manifest_a_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + + # Add manifest_b to the repo + manifest_b = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_b&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + manifest_b_digest = self.client.get(manifest_b)['digest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_b]}) + # Tag manifest_b as `manifest_a` + params = { + 'tag': "manifest_a", + 'repository': self.to_repo['pulp_href'], + 'digest': manifest_b_digest + } + self.client.post(DOCKER_TAGGING_PATH, params) + + # Now add original manifest_a tag to the repo, which should remove the + # new manifest_a tag, but leave the tagged manifest (manifest_b) + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a_tag]}) + + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['removed']['docker.tag']['count'], 1) + self.assertFalse('docker.manifest' in latest['content_summary']['removed']) + self.assertFalse('docker.blob' in latest['content_summary']['removed']) + + def test_many_tagged_manifest_lists(self): + """Add several Manifest List, related manifests, and related blobs.""" + ml_i_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + ml_ii_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_ii&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + ml_iii_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_iii&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + ml_iv_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_iv&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + { + 'repository': self.to_repo['pulp_href'], + 'content_units': [ml_i_tag, ml_ii_tag, ml_iii_tag, ml_iv_tag] + } + ) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 4) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 9) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 10) diff --git a/pulp_docker/tests/functional/api/test_recursive_remove.py b/pulp_docker/tests/functional/api/test_recursive_remove.py new file mode 100644 index 00000000..229d0101 --- /dev/null +++ b/pulp_docker/tests/functional/api/test_recursive_remove.py @@ -0,0 +1,334 @@ +# # coding=utf-8 +"""Tests that recursively remove docker content from repositories.""" +import unittest + +from pulp_smash import api, config +from pulp_smash.pulp3.constants import REPO_PATH +from pulp_smash.pulp3.utils import gen_repo, sync +from requests.exceptions import HTTPError + +from pulp_docker.tests.functional.constants import ( + DOCKER_TAG_PATH, + DOCKER_REMOTE_PATH, + DOCKER_RECURSIVE_ADD_PATH, + DOCKER_RECURSIVE_REMOVE_PATH, + DOCKERHUB_PULP_FIXTURE_1, +) +from pulp_docker.tests.functional.utils import gen_docker_remote + + +class TestRecursiveRemove(unittest.TestCase): + """ + Test recursively removing docker content from a repository. + + This test targets the follow feature: + https://pulp.plan.io/issues/5179 + """ + + @classmethod + def setUpClass(cls): + """Sync pulp/test-fixture-1 so we can copy content from it.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.from_repo = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + sync(cls.cfg, cls.remote, cls.from_repo) + latest_version = cls.client.get(cls.from_repo['pulp_href'])['latest_version_href'] + cls.latest_from_version = "repository_version={version}".format(version=latest_version) + + def setUp(self): + """Create an empty repository to copy into.""" + self.to_repo = self.client.post(REPO_PATH, gen_repo()) + self.addCleanup(self.client.delete, self.to_repo['pulp_href']) + + @classmethod + def tearDownClass(cls): + """Delete things made in setUpClass. addCleanup feature does not work with setupClass.""" + cls.client.delete(cls.from_repo['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + + def test_missing_repository_argument(self): + """Ensure Repository argument is required.""" + with self.assertRaises(HTTPError) as context: + self.client.post(DOCKER_RECURSIVE_ADD_PATH) + self.assertEqual(context.exception.response.status_code, 400) + + def test_repository_only(self): + """Passing only a repository creates a new version.""" + # Create a new version, repository must have latest version to be valid. + self.client.post(DOCKER_RECURSIVE_ADD_PATH, {'repository': self.to_repo['pulp_href']}) + after_add_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + + # Actual test + self.client.post(DOCKER_RECURSIVE_REMOVE_PATH, {'repository': self.to_repo['pulp_href']}) + after_remove_version_href = self.client.get( + self.to_repo['pulp_href'])['latest_version_href'] + self.assertNotEqual(after_add_version_href, after_remove_version_href) + latest = self.client.get(after_remove_version_href) + for content_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertFalse(content_type in latest['content_summary']['removed'], msg=content_type) + + def test_repository_only_no_latest_version(self): + """Create a new version, even when there is nothing to remove.""" + self.client.post(DOCKER_RECURSIVE_REMOVE_PATH, {'repository': self.to_repo['pulp_href']}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + self.assertIsNotNone(latest_version_href) + latest = self.client.get(latest_version_href) + for content_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertFalse(content_type in latest['content_summary']['removed'], msg=content_type) + + def test_manifest_recursion(self): + """Add a manifest and its related blobs.""" + manifest_a = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + # Ensure test begins in the correct state + self.assertFalse('docker.tag' in latest['content_summary']['added']) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 2) + + # Actual test + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertFalse('docker.tag' in latest['content_summary']['removed']) + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 2) + + def test_manifest_list_recursion(self): + """Add a Manifest List, related manifests, and related blobs.""" + ml_i = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + # Ensure test begins in the correct state + self.assertFalse('docker.tag' in latest['content_summary']['added']) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 3) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 4) + + # Actual test + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertFalse('docker.tag' in latest['content_summary']['removed']) + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 3) + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 4) + + def test_tagged_manifest_list_recursion(self): + """Add a tagged manifest list, and its related manifests and blobs.""" + ml_i_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i_tag]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + # Ensure test begins in the correct state + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 3) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 4) + + # Actual test + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i_tag]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertEqual(latest['content_summary']['removed']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 3) + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 4) + + def test_tagged_manifest_recursion(self): + """Add a tagged manifest and its related blobs.""" + manifest_a_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a_tag]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + # Ensure valid starting state + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 2) + + # Actual test + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a_tag]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + self.assertEqual(latest['content_summary']['removed']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 2) + + def test_manifests_shared_blobs(self): + """Starting with 2 manifests that share blobs, remove one of them.""" + manifest_a = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + manifest_e = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_e&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_a, manifest_e]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + # Ensure valid starting state + self.assertFalse('docker.tag' in latest['content_summary']['added']) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 2) + # manifest_a has 1 blob, 1 config blob, and manifest_e has 2 blob 1 config blob + # manifest_a blob is shared with manifest_e + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 4) + + # Actual test + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [manifest_e]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertFalse('docker.tag' in latest['content_summary']['removed']) + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 1) + # Despite having 3 blobs, only 2 are removed, 1 is shared with manifest_a. + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 2) + + def test_manifest_lists_shared_manifests(self): + """Starting with 2 manifest lists that share a manifest, remove one of them.""" + ml_i = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + # Shares 1 manifest with ml_i + ml_iii = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_iii&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['tagged_manifest'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_i, ml_iii]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + # Ensure valid starting state + self.assertFalse('docker.tag' in latest['content_summary']['added']) + # 2 manifest lists, each with 2 manifests, 1 manifest shared + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 5) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 6) + + # Actual test + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + {'repository': self.to_repo['pulp_href'], 'content_units': [ml_iii]}) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertFalse('docker.tag' in latest['content_summary']['removed']) + # 1 manifest list, 1 manifest + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 2) + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 2) + + def test_many_tagged_manifest_lists(self): + """Add several Manifest List, related manifests, and related blobs.""" + ml_i_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_i&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + ml_ii_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_ii&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + ml_iii_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_iii&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + ml_iv_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=ml_iv&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0]['pulp_href'] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + { + 'repository': self.to_repo['pulp_href'], + 'content_units': [ml_i_tag, ml_ii_tag, ml_iii_tag, ml_iv_tag] + } + ) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 4) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 9) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 10) + + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + { + 'repository': self.to_repo['pulp_href'], + 'content_units': [ml_i_tag, ml_ii_tag, ml_iii_tag, ml_iv_tag] + } + ) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + + self.assertEqual(latest['content_summary']['removed']['docker.tag']['count'], 4) + self.assertEqual(latest['content_summary']['removed']['docker.manifest']['count'], 9) + self.assertEqual(latest['content_summary']['removed']['docker.blob']['count'], 10) + + def test_cannot_remove_tagged_manifest(self): + """ + Try to remove a manifest (without removing tag). Creates a new version, but nothing removed. + """ + manifest_a_tag = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a&{v_filter}".format(v_filter=self.latest_from_version), + ))['results'][0] + self.client.post( + DOCKER_RECURSIVE_ADD_PATH, + { + 'repository': self.to_repo['pulp_href'], + 'content_units': [manifest_a_tag['pulp_href']] + } + ) + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + self.assertEqual(latest['content_summary']['added']['docker.tag']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.manifest']['count'], 1) + self.assertEqual(latest['content_summary']['added']['docker.blob']['count'], 2) + + self.client.post( + DOCKER_RECURSIVE_REMOVE_PATH, + { + 'repository': self.to_repo['pulp_href'], + 'content_units': [manifest_a_tag['tagged_manifest']] + } + ) + + latest_version_href = self.client.get(self.to_repo['pulp_href'])['latest_version_href'] + latest = self.client.get(latest_version_href) + for content_type in ['docker.tag', 'docker.manifest', 'docker.blob']: + self.assertFalse(content_type in latest['content_summary']['removed'], msg=content_type) diff --git a/pulp_docker/tests/functional/api/test_sync.py b/pulp_docker/tests/functional/api/test_sync.py new file mode 100644 index 00000000..1d69bed2 --- /dev/null +++ b/pulp_docker/tests/functional/api/test_sync.py @@ -0,0 +1,154 @@ +# coding=utf-8 +"""Tests that sync docker plugin repositories.""" +import unittest + +from pulp_smash import api, cli, config, exceptions +from pulp_smash.pulp3.constants import MEDIA_PATH, REPO_PATH +from pulp_smash.pulp3.utils import delete_orphans, gen_repo, sync + +from pulp_docker.tests.functional.constants import ( + DOCKER_TAG_PATH, + DOCKER_REMOTE_PATH, + DOCKERHUB_PULP_FIXTURE_1, +) +from pulp_docker.tests.functional.utils import gen_docker_remote +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + + +class BasicSyncTestCase(unittest.TestCase): + """Sync repositories with the docker plugin.""" + + maxDiff = None + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + + def test_sync(self): + """Sync repositories with the docker plugin. + + In order to sync a repository a remote has to be associated within + this repository. When a repository is created this version field is set + as None. After a sync the repository version is updated. + + Do the following: + + 1. Create a repository, and a remote. + 2. Assert that repository version is None. + 3. Sync the remote. + 4. Assert that repository version is not None. + 5. Sync the remote one more time. + 6. Assert that repository version is different from the previous one. + """ + repo = self.client.post(REPO_PATH, gen_repo()) + self.addCleanup(self.client.delete, repo['pulp_href']) + + remote = self.client.post(DOCKER_REMOTE_PATH, gen_docker_remote()) + self.addCleanup(self.client.delete, remote['pulp_href']) + + # Sync the repository. + self.assertIsNone(repo['latest_version_href']) + sync(self.cfg, remote, repo) + repo = self.client.get(repo['pulp_href']) + self.assertIsNotNone(repo['latest_version_href']) + + # Sync the repository again. + latest_version_href = repo['latest_version_href'] + sync(self.cfg, remote, repo) + repo = self.client.get(repo['pulp_href']) + self.assertNotEqual(latest_version_href, repo['latest_version_href']) + + def test_file_decriptors(self): + """Test whether file descriptors are closed properly. + + This test targets the following issue: + `Pulp #4073 `_ + + Do the following: + 1. Check if 'lsof' is installed. If it is not, skip this test. + 2. Create and sync a repo. + 3. Run the 'lsof' command to verify that files in the + path ``/var/lib/pulp/`` are closed after the sync. + 4. Assert that issued command returns `0` opened files. + """ + cli_client = cli.Client(self.cfg, cli.echo_handler) + + # check if 'lsof' is available + if cli_client.run(('which', 'lsof')).returncode != 0: + raise unittest.SkipTest('lsof package is not present') + + repo = self.client.post(REPO_PATH, gen_repo()) + self.addCleanup(self.client.delete, repo['pulp_href']) + + remote = self.client.post(DOCKER_REMOTE_PATH, gen_docker_remote()) + self.addCleanup(self.client.delete, remote['pulp_href']) + + sync(self.cfg, remote, repo) + + cmd = 'lsof -t +D {}'.format(MEDIA_PATH).split() + response = cli_client.run(cmd).stdout + self.assertEqual(len(response), 0, response) + + +class SyncInvalidURLTestCase(unittest.TestCase): + """Sync a repository with an invalid url on the Remote.""" + + def test_all(self): + """ + Sync a repository using a Remote url that does not exist. + + Test that we get a task failure. + + """ + cfg = config.get_config() + client = api.Client(cfg, api.json_handler) + + repo = client.post(REPO_PATH, gen_repo()) + self.addCleanup(client.delete, repo['pulp_href']) + + remote = client.post( + DOCKER_REMOTE_PATH, + gen_docker_remote(url="http://i-am-an-invalid-url.com/invalid/") + ) + self.addCleanup(client.delete, remote['pulp_href']) + + with self.assertRaises(exceptions.TaskReportError): + sync(cfg, remote, repo) + + +class TestRepeatedSync(unittest.TestCase): + """Test behavior when a sync is repeated.""" + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + cls.from_repo = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + delete_orphans(cls.cfg) + + @classmethod + def tearDownClass(cls): + """Delete things made in setUpClass. addCleanup feature does not work with setupClass.""" + cls.client.delete(cls.from_repo['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + delete_orphans(cls.cfg) + + def test_sync_idempotency(self): + """Ensure that sync does not create orphan tags https://pulp.plan.io/issues/5252 .""" + sync(self.cfg, self.remote, self.from_repo) + first_sync_tags_named_a = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a", + )) + sync(self.cfg, self.remote, self.from_repo) + second_sync_tags_named_a = self.client.get("{unit_path}?{filters}".format( + unit_path=DOCKER_TAG_PATH, + filters="name=manifest_a", + )) + self.assertEqual(first_sync_tags_named_a['count'], 1) + self.assertEqual(second_sync_tags_named_a['count'], 1) diff --git a/pulp_docker/tests/functional/api/test_tagging_images.py b/pulp_docker/tests/functional/api/test_tagging_images.py new file mode 100644 index 00000000..fd6aa4bc --- /dev/null +++ b/pulp_docker/tests/functional/api/test_tagging_images.py @@ -0,0 +1,198 @@ +# coding=utf-8 +"""Tests for tagging and untagging images.""" +import unittest + +from pulp_smash import api, config +from pulp_smash.pulp3.utils import gen_repo, sync +from pulp_smash.pulp3.constants import REPO_PATH + +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 +from pulp_docker.tests.functional.utils import gen_docker_remote + +from pulp_docker.tests.functional.constants import ( + DOCKER_TAG_PATH, + DOCKER_TAGGING_PATH, + DOCKER_REMOTE_PATH, + DOCKERHUB_PULP_FIXTURE_1, + DOCKER_UNTAGGING_PATH +) +from requests import HTTPError + + +class TaggingTestCase(unittest.TestCase): + """Test case for tagging and untagging images.""" + + @classmethod + def setUpClass(cls): + """Create class wide-variables.""" + cls.cfg = config.get_config() + cls.client = api.Client(cls.cfg, api.json_handler) + + cls.repository = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + sync(cls.cfg, cls.remote, cls.repository) + + @classmethod + def tearDownClass(cls): + """Clean generated resources.""" + cls.client.delete(cls.repository['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + + def test_01_tag_first_image(self): + """ + Create a new test for manifest. + + This test checks if the tag was created in a new repository version. + """ + manifest_a = self.get_manifest_by_tag('manifest_a') + self.tag_image(manifest_a, 'new_tag') + + new_repository_version_href = '{repository_href}versions/{new_version}/'.format( + repository_href=self.repository['pulp_href'], + new_version='2' + ) + + added_tags_href = '{unit_path}?{filters}'.format( + unit_path=DOCKER_TAG_PATH, + filters=f'repository_version_added={new_repository_version_href}' + ) + created_tag = self.client.get(added_tags_href)['results'][0] + self.assertEqual(created_tag['name'], 'new_tag', created_tag['name']) + + repository_version = self.client.get(new_repository_version_href) + + added_content = repository_version['content_summary']['added'] + added_tags = added_content['docker.tag']['count'] + self.assertEqual(added_tags, 1, added_content) + + removed_content = repository_version['content_summary']['removed'] + self.assertEqual(removed_content, {}, removed_content) + + def test_02_tag_first_image_with_same_tag(self): + """ + Tag the same manifest with the same name. + + This test checks if a new repository version was created with no content added. + """ + manifest_a = self.get_manifest_by_tag('manifest_a') + self.tag_image(manifest_a, 'new_tag') + + new_repository_version_href = '{repository_href}versions/{new_version}/'.format( + repository_href=self.repository['pulp_href'], + new_version='3' + ) + + repository_version = self.client.get(new_repository_version_href) + + added_content = repository_version['content_summary']['added'] + self.assertEqual(added_content, {}, added_content) + + removed_content = repository_version['content_summary']['removed'] + self.assertEqual(removed_content, {}, removed_content) + + def test_03_tag_second_image_with_same_tag(self): + """ + Tag a different manifest with the same name. + + This test checks if a new repository version was created with a new content added + and the old removed. + """ + manifest_a = self.get_manifest_by_tag('manifest_a') + manifest_b = self.get_manifest_by_tag('manifest_b') + self.tag_image(manifest_b, 'new_tag') + + new_repository_version_href = '{repository_href}versions/{new_version}/'.format( + repository_href=self.repository['pulp_href'], + new_version='4' + ) + + added_tags_href = '{unit_path}?{filters}'.format( + unit_path=DOCKER_TAG_PATH, + filters=f'repository_version_added={new_repository_version_href}' + ) + created_tag = self.client.get(added_tags_href)['results'][0] + self.assertEqual(created_tag['name'], 'new_tag', created_tag['name']) + + created_tag_manifest = self.client.get(created_tag['tagged_manifest']) + self.assertEqual(created_tag_manifest, manifest_b, created_tag_manifest) + + removed_tags_href = '{unit_path}?{filters}'.format( + unit_path=DOCKER_TAG_PATH, + filters=f'repository_version_removed={new_repository_version_href}' + ) + removed_tag = self.client.get(removed_tags_href)['results'][0] + self.assertEqual(removed_tag['name'], 'new_tag', removed_tag['name']) + + removed_tag_manifest = self.client.get(removed_tag['tagged_manifest']) + self.assertEqual(removed_tag_manifest, manifest_a, removed_tag_manifest) + + repository_version = self.client.get(new_repository_version_href) + + added_content = repository_version['content_summary']['added'] + added_tags = added_content['docker.tag']['count'] + self.assertEqual(added_tags, 1, added_content) + + removed_content = repository_version['content_summary']['removed'] + removed_tags = removed_content['docker.tag']['count'] + self.assertEqual(removed_tags, 1, removed_content) + + def test_04_untag_second_image(self): + """Untag the manifest and check if the tag was added in a new repository version.""" + self.untag_image('new_tag') + + new_repository_version_href = '{repository_href}versions/{new_version}/'.format( + repository_href=self.repository['pulp_href'], + new_version='5' + ) + + removed_tags_href = '{unit_path}?{filters}'.format( + unit_path=DOCKER_TAG_PATH, + filters=f'repository_version_removed={new_repository_version_href}' + ) + + repository_version = self.client.get(new_repository_version_href) + + removed_content = repository_version['content_summary']['removed'] + removed_tags = removed_content['docker.tag']['href'] + self.assertEqual(removed_tags, removed_tags_href, removed_tags) + + added_content = repository_version['content_summary']['added'] + self.assertEqual(added_content, {}, added_content) + + removed_tag = self.client.get(removed_tags_href)['results'][0] + self.assertEqual(removed_tag['name'], 'new_tag', removed_tag) + + def test_05_untag_second_image_again(self): + """Untag the manifest that was already untagged.""" + with self.assertRaises(HTTPError): + self.untag_image('new_tag') + + def get_manifest_by_tag(self, tag_name): + """Fetch a manifest by the tag name.""" + latest_version = self.client.get( + self.repository['pulp_href'] + )['latest_version_href'] + + manifest_a_href = self.client.get('{unit_path}?{filters}'.format( + unit_path=DOCKER_TAG_PATH, + filters=f'name={tag_name}&repository_version={latest_version}' + ))['results'][0]['tagged_manifest'] + return self.client.get(manifest_a_href) + + def tag_image(self, manifest, tag_name): + """Perform a tagging operation.""" + params = { + 'tag': tag_name, + 'repository': self.repository['pulp_href'], + 'digest': manifest['digest'] + } + self.client.post(DOCKER_TAGGING_PATH, params) + + def untag_image(self, tag_name): + """Perform an untagging operation.""" + params = { + 'tag': tag_name, + 'repository': self.repository['pulp_href'] + } + self.client.post(DOCKER_UNTAGGING_PATH, params) diff --git a/pulp_docker/tests/functional/api/test_token_authentication.py b/pulp_docker/tests/functional/api/test_token_authentication.py new file mode 100644 index 00000000..a9d7947b --- /dev/null +++ b/pulp_docker/tests/functional/api/test_token_authentication.py @@ -0,0 +1,162 @@ +# coding=utf-8 +"""Tests for token authentication.""" +import unittest + +from urllib.parse import urljoin +from requests.exceptions import HTTPError +from requests.auth import AuthBase + +from pulp_smash import api, config, cli +from pulp_smash.pulp3.utils import gen_repo, sync, gen_distribution +from pulp_smash.pulp3.constants import REPO_PATH + +from pulp_docker.tests.functional.utils import set_up_module as setUpModule # noqa:F401 +from pulp_docker.tests.functional.utils import gen_docker_remote + +from pulp_docker.tests.functional.constants import ( + DOCKER_TAG_PATH, + DOCKER_REMOTE_PATH, + DOCKERHUB_PULP_FIXTURE_1, + DOCKER_DISTRIBUTION_PATH +) +from pulp_docker.constants import MEDIA_TYPE + + +""" +@unittest.skip( + "A handler for a token authentication relies on a provided token server's " + "fully qualified domain name (TOKEN_SERVER) in the file /etc/pulp/settings.py; " + "therefore, it is necessary to check if TOKEN_SERVER was specified; otherwise, " + "these tests are no longer valid because the token authentication is turned off " + "by default." +) +""" + + +class TokenAuthenticationTestCase(unittest.TestCase): + """ + A test case for authenticating users via Bearer token. + + This tests targets the following issue: + + * `Pulp #4938 `_ + """ + + @classmethod + def setUpClass(cls): + """Create class wide-variables.""" + cls.cfg = config.get_config() + + token_auth = cls.cfg.hosts[0].roles['token auth'] + client = cli.Client(cls.cfg) + client.run('openssl ecparam -genkey -name prime256v1 -noout -out {}' + .format(token_auth['private key']).split()) + client.run('openssl ec -in {} -pubout -out {}'.format( + token_auth['private key'], token_auth['public key']).split()) + + cls.client = api.Client(cls.cfg, api.page_handler) + + cls.repository = cls.client.post(REPO_PATH, gen_repo()) + remote_data = gen_docker_remote(upstream_name=DOCKERHUB_PULP_FIXTURE_1) + cls.remote = cls.client.post(DOCKER_REMOTE_PATH, remote_data) + sync(cls.cfg, cls.remote, cls.repository) + + cls.distribution = cls.client.using_handler(api.task_handler).post( + DOCKER_DISTRIBUTION_PATH, + gen_distribution(repository=cls.repository['pulp_href']) + ) + + @classmethod + def tearDownClass(cls): + """Clean generated resources.""" + cls.client.delete(cls.repository['pulp_href']) + cls.client.delete(cls.remote['pulp_href']) + cls.client.delete(cls.distribution['pulp_href']) + + def test_pull_image_with_raw_http_requests(self): + """ + Test if a content was pulled from a registry by using raw HTTP requests. + + The registry offers a reference to a certified authority which generates a + Bearer token. The generated Bearer token is afterwards used to pull the image. + All requests are sent via aiohttp modules. + """ + image_path = '/v2/{}/manifests/{}'.format(self.distribution['base_path'], 'manifest_a') + latest_image_url = urljoin(self.cfg.get_content_host_base_url(), image_path) + + with self.assertRaises(HTTPError) as cm: + self.client.get(latest_image_url, headers={'Accept': MEDIA_TYPE.MANIFEST_V2}) + + content_response = cm.exception.response + self.assertEqual(content_response.status_code, 401) + + authenticate_header = content_response.headers['Www-Authenticate'] + queries = AuthenticationHeaderQueries(authenticate_header) + content_response = self.client.get( + queries.realm, + params={'service': queries.service, 'scope': queries.scope} + ) + content_response = self.client.get( + latest_image_url, + auth=BearerTokenAuth(content_response['token']), + headers={'Accept': MEDIA_TYPE.MANIFEST_V2} + ) + self.compare_config_blob_digests(content_response['config']['digest']) + + def test_pull_image_with_real_docker_client(self): + """ + Test if a CLI client is able to pull an image from an authenticated registry. + + This test checks if ordinary clients, like docker, or podman, are able to pull the + image from a secured registry. + """ + registry = cli.RegistryClient(self.cfg) + registry.raise_if_unsupported(unittest.SkipTest, 'Test requires podman/docker') + + image_url = urljoin( + self.cfg.get_content_host_base_url(), + self.distribution['base_path'] + ) + image_with_tag = f'{image_url}:manifest_a' + registry.pull(image_with_tag) + + image = registry.inspect(image_with_tag) + self.compare_config_blob_digests(image[0]['Id']) + + def compare_config_blob_digests(self, pulled_manifest_digest): + """Check if a valid config was pulled from a registry.""" + tags_by_name_url = f'{DOCKER_TAG_PATH}?name=manifest_a' + tag_response = self.client.get(tags_by_name_url) + + tagged_manifest_href = tag_response[0]['tagged_manifest'] + manifest_response = self.client.get(tagged_manifest_href) + + config_blob_response = self.client.get(manifest_response['config_blob']) + self.assertEqual(pulled_manifest_digest, config_blob_response['digest']) + + +class AuthenticationHeaderQueries: + """A data class to store header queries located in the Www-Authenticate header.""" + + def __init__(self, authenticate_header): + """Extract service, realm, and scope from the header.""" + realm, service, scope = authenticate_header[7:].split(',') + # realm="rlm" -> rlm + self.realm = realm[6:][1:-1] + # service="srv" -> srv + self.service = service[8:][1:-1] + # scope="scp" -> scp + self.scope = scope[6:][1:-1] + + +class BearerTokenAuth(AuthBase): + """A subclass for building a JWT Authorization header out of a provided token.""" + + def __init__(self, token): + """Store a Bearer token that is going to be used in the request object.""" + self.token = token + + def __call__(self, r): + """Attaches a Bearer token authentication to the given request object.""" + r.headers['Authorization'] = 'Bearer {}'.format(self.token) + return r diff --git a/pulp_docker/tests/functional/constants.py b/pulp_docker/tests/functional/constants.py new file mode 100644 index 00000000..b6ce38e7 --- /dev/null +++ b/pulp_docker/tests/functional/constants.py @@ -0,0 +1,84 @@ +# coding=utf-8 +from urllib.parse import urljoin + +from pulp_smash.constants import PULP_FIXTURES_BASE_URL +from pulp_smash.pulp3.constants import ( + BASE_PATH, + BASE_DISTRIBUTION_PATH, + BASE_REMOTE_PATH, + CONTENT_PATH +) + +DOCKER_MANIFEST_PATH = urljoin(CONTENT_PATH, 'docker/manifests/') + +DOCKER_TAG_PATH = urljoin(CONTENT_PATH, 'docker/tags/') + +DOCKER_BLOB_PATH = urljoin(CONTENT_PATH, 'docker/blobs/') + +DOCKER_CONTENT_PATH = urljoin(CONTENT_PATH, 'docker/unit/') + +DOCKER_TAGGING_PATH = urljoin(BASE_PATH, 'docker/tag/') + +DOCKER_UNTAGGING_PATH = urljoin(BASE_PATH, 'docker/untag/') + +DOCKER_TAG_COPY_PATH = urljoin(BASE_PATH, 'docker/tags/copy/') + +DOCKER_MANIFEST_COPY_PATH = urljoin(BASE_PATH, 'docker/manifests/copy/') + +DOCKER_CONTENT_NAME = 'docker.blob' + +DOCKER_DISTRIBUTION_PATH = urljoin(BASE_DISTRIBUTION_PATH, 'docker/docker/') + +DOCKER_REMOTE_PATH = urljoin(BASE_REMOTE_PATH, 'docker/docker/') + +DOCKER_RECURSIVE_ADD_PATH = urljoin(BASE_PATH, 'docker/recursive-add/') + +DOCKER_RECURSIVE_REMOVE_PATH = urljoin(BASE_PATH, 'docker/recursive-remove/') + +DOCKER_IMAGE_URL = urljoin(PULP_FIXTURES_BASE_URL, 'docker/busybox:latest.tar') +"""The URL to a Docker image as created by ``docker save``.""" + +# hello-world is the smalest docker image available on docker hub 1.84kB +DOCKER_UPSTREAM_NAME = 'hello-world' +"""The name of a Docker repository. + +This repository has several desireable properties: + +* It is available via both :data:`DOCKER_V1_FEED_URL` and + :data:`DOCKER_V2_FEED_URL`. +* It has a manifest list, where one entry has an architecture of amd64 and an + os of linux. (The "latest" tag offers this.) +* It is relatively small. + +This repository also has several shortcomings: + +* This repository isn't an official repository. It's less trustworthy, and may + be more likely to change with little or no notice. +* It doesn't have a manifest list where no list entries have an architecture of + amd64 and an os of linux. (The "arm32v7" tag provides schema v1 content.) + +One can get a high-level view of the content in this repository by executing: + +.. code-block:: sh + + curl --location --silent \ + https://registry.hub.docker.com/v2/repositories/$this_constant/tags \ + | python -m json.tool +""" + +DOCKER_UPSTREAM_TAG = ":linux" +"""Alternative tag for the DOCKER_UPSTREAM_NAME image.""" + +DOCKER_V1_FEED_URL = 'https://index.docker.io' +"""The URL to a V1 Docker registry. + +This URL can be used as the "feed" property of a Pulp Docker registry. +""" + +DOCKER_V2_FEED_URL = 'https://registry-1.docker.io' +"""The URL to a V2 Docker registry. + +This URL can be used as the "feed" property of a Pulp Docker registry. +""" + +DOCKERHUB_PULP_FIXTURE_1 = 'pulp/test-fixture-1' diff --git a/pulp_docker/tests/functional/test_convert.py b/pulp_docker/tests/functional/test_convert.py new file mode 100644 index 00000000..3402f125 --- /dev/null +++ b/pulp_docker/tests/functional/test_convert.py @@ -0,0 +1,110 @@ +import base64 + +from jwkest import jws + +from pulp_docker.app import docker_convert + + +class Test: + """Converter_s2_to_s1 test class""" + + def test_convert(self): + """Test schema converter on a known manifest""" + cnv = docker_convert.ConverterS2toS1(MANIFEST, CONFIG_LAYER, 'test-repo', 'tes-tag') + signed_mf = cnv.convert() + validate_signature(signed_mf) + + empty = dict(blobSum=cnv.EMPTY_LAYER) + assert [dict(blobSum="sha256:layer1"), empty, empty, empty, + dict(blobSum="sha256:base")] == cnv.fs_layers + + def test_compute_layers(self): + """Test that computing the layers produces the expected data""" + cnv = docker_convert.ConverterS2toS1(MANIFEST, CONFIG_LAYER, 'test-repo', 'tes-tag') + cnv.compute_layers() + empty = dict(blobSum=cnv.EMPTY_LAYER) + assert [dict(blobSum="sha256:layer1"), empty, empty, empty, + dict(blobSum="sha256:base")] == cnv.fs_layers + assert [ + {'v1Compatibility': '{"architecture":"amd64","author":"Mihai Ibanescu ","config":{"Cmd":["/bin/bash"],"Hostname":"decafbad"},"container_config":{"Hostname":"decafbad","Tty":false},"created":"2019-09-05T21:28:52.173079282Z","docker_version":"1.13.1","id":"d7b329ed9d186ff20c25399e848116430ee9b6ae022cb9f3dc3406144ec3685d","parent":"6474547c15d178825c70a42efdc59a88c6e30d764d184b415f32484562803446"}'}, # noqa + {'v1Compatibility': '{"container_config":{"Cmd":"/bin/sh -c #(nop) MAINTAINER Mihai Ibanescu "},"created":"2019-09-05T21:28:43.305854958Z","id":"6474547c15d178825c70a42efdc59a88c6e30d764d184b415f32484562803446","parent":"5708420291e0a86d8dc08ec40b2c1b1799117c33fe85032b87227632f70c1018","throwaway":true}'}, # noqa + {'v1Compatibility': '{"container_config":{"Cmd":"/bin/sh -c #(nop) CMD [\\"/bin/bash\\"]"},"created":"2018-03-06T00:48:12.679169547Z","id":"5708420291e0a86d8dc08ec40b2c1b1799117c33fe85032b87227632f70c1018","parent":"9e9220abceaf86f2ad7820ae8124d01223d8ec022b9a6cb8c99a8ae1747137ea","throwaway":true}'}, # noqa + {'v1Compatibility': '{"container_config":{"Cmd":"/bin/sh -c #(nop) LABEL name=CentOS Base Image vendor=CentOS license=GPLv2 build-date=20180302"},"created":"2018-03-06T00:48:12.458578213Z","id":"9e9220abceaf86f2ad7820ae8124d01223d8ec022b9a6cb8c99a8ae1747137ea","parent":"cb48c1db9c0a1ede7c85c85351856fc3e40e750931295c8fac837c63b403586a","throwaway":true}'}, # noqa + {'v1Compatibility': '{"container_config":{"Cmd":"/bin/sh -c #(nop) ADD file:FILE_CHECKSUM in / "},"created":"2018-03-06T00:48:12.077095981Z","id":"cb48c1db9c0a1ede7c85c85351856fc3e40e750931295c8fac837c63b403586a"}'}, # noqa + ] == cnv.history + + +def validate_signature(signed_mf): + """ + Validate the signature of a signed manifest + + A signed manifest is a JSON document with a signature attribute + as the last element. + """ + # In order to validate the signature, we need the exact original payload + # (the document without the signature). We cannot json.load the document + # and get rid of the signature, the payload would likely end up + # differently because of differences in field ordering and indentation. + # So we need to strip the signature using plain string manipulation, and + # add back a trailing } + + # strip the signature block + payload, sep, signatures = signed_mf.partition(' "signatures"') + # get rid of the trailing ,\n, and add \n} + jw_payload = payload[:-2] + '\n}' + # base64-encode and remove any trailing = + jw_payload = base64.urlsafe_b64encode(jw_payload.encode('ascii')).decode('ascii').rstrip("=") + # add payload as a json attribute, and then add the signatures back + complete_msg = payload + ' "payload": "{}",\n'.format(jw_payload) + sep + signatures + _jws = jws.JWS() + _jws.verify_json(complete_msg.encode('ascii')) + + +MANIFEST = dict(schemaVersion=2, layers=[ + dict(digest="sha256:base"), + dict(digest="sha256:layer1"), +]) + +CONFIG_LAYER = dict( + architecture="amd64", + author="Mihai Ibanescu ", + config=dict(Hostname="decafbad", Cmd=["/bin/bash"]), + container_config=dict(Hostname="decafbad", Tty=False), + created="2019-09-05T21:28:52.173079282Z", + docker_version="1.13.1", + history=[ + { + "created": "2018-03-06T00:48:12.077095981Z", + "created_by": "/bin/sh -c #(nop) ADD file:FILE_CHECKSUM in / " + }, + { + "created": "2018-03-06T00:48:12.458578213Z", + "created_by": "/bin/sh -c #(nop) LABEL name=CentOS Base Image vendor=CentOS " + "license=GPLv2 build-date=20180302", + "empty_layer": True + }, + { + "created": "2018-03-06T00:48:12.679169547Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/bin/bash\"]", + "empty_layer": True + }, + { + "created": "2019-09-05T21:28:43.305854958Z", + "author": "Mihai Ibanescu ", + "created_by": "/bin/sh -c #(nop) MAINTAINER Mihai Ibanescu ", + "empty_layer": True + }, + { + "created": "2019-09-05T21:28:52.173079282Z", + "author": "Mihai Ibanescu ", + "created_by": "/bin/sh -c touch /usr/share/dummy.txt" + }, + ], + rootfs={ + "type": "layers", + "diff_ids": [ + "sha256:uncompressed_base", + "sha256:uncompressed_layer1" + ], + }, +) diff --git a/pulp_docker/tests/functional/utils.py b/pulp_docker/tests/functional/utils.py new file mode 100644 index 00000000..d0864753 --- /dev/null +++ b/pulp_docker/tests/functional/utils.py @@ -0,0 +1,124 @@ +# coding=utf-8 +"""Utilities for tests for the docker plugin.""" +import requests +from functools import partial +from unittest import SkipTest + +from pulp_smash import api, selectors +from pulp_smash.pulp3.constants import ( + REPO_PATH +) +from pulp_smash.pulp3.utils import ( + gen_remote, + gen_repo, + get_content, + require_pulp_3, + require_pulp_plugins, + sync +) + +from pulp_docker.tests.functional.constants import ( + DOCKER_CONTENT_NAME, + DOCKER_CONTENT_PATH, + DOCKER_REMOTE_PATH, + DOCKER_UPSTREAM_NAME, + DOCKER_V2_FEED_URL, +) + + +def set_up_module(): + """Skip tests Pulp 3 isn't under test or if pulp_docker isn't installed.""" + require_pulp_3(SkipTest) + require_pulp_plugins({'pulp_docker'}, SkipTest) + + +def gen_docker_remote(**kwargs): + """Generate dict with common remote properties.""" + return gen_remote( + kwargs.pop('url', DOCKER_V2_FEED_URL), + upstream_name=kwargs.pop('upstream_name', DOCKER_UPSTREAM_NAME), + **kwargs + ) + + +def get_docker_hub_remote_blobsums(upstream_name=DOCKER_UPSTREAM_NAME): + """Get remote blobsum list from dockerhub registry.""" + token_url = ( + 'https://auth.docker.io/token' + '?service=registry.docker.io' + '&scope=repository:library/{0}:pull' + ).format(upstream_name) + token_response = requests.get(token_url) + token_response.raise_for_status() + token = token_response.json()['token'] + + blob_url = ( + '{0}/v2/library/{1}/manifests/latest' + ).format(DOCKER_V2_FEED_URL, upstream_name) + response = requests.get( + blob_url, + headers={'Authorization': 'Bearer ' + token} + ) + response.raise_for_status() + return response.json()['fsLayers'] + + +def get_docker_image_paths(repo, version_href=None): + """Return the relative path of content units present in a file repository. + + :param repo: A dict of information about the repository. + :param version_href: The repository version to read. + :returns: A list with the paths of units present in a given repository. + """ + return [ + content_unit['_artifact'] + for content_unit + in get_content(repo, version_href)[DOCKER_CONTENT_NAME] + ] + + +def gen_docker_image_attrs(artifact): + """Generate a dict with content unit attributes. + + :param: artifact: A dict of info about the artifact. + :returns: A semi-random dict for use in creating a content unit. + """ + # FIXME: Add content specific metadata here. + return {'_artifact': artifact['pulp_href']} + + +def populate_pulp(cfg, url=DOCKER_V2_FEED_URL): + """Add docker contents to Pulp. + + :param pulp_smash.config.PulpSmashConfig: Information about a Pulp application. + :param url: The docker repository URL. Defaults to + :data:`pulp_smash.constants.DOCKER_FIXTURE_URL` + :returns: A list of dicts, where each dict describes one file content in Pulp. + """ + client = api.Client(cfg, api.json_handler) + remote = {} + repo = {} + try: + remote.update( + client.post( + DOCKER_REMOTE_PATH, + gen_remote(url, upstream_name=DOCKER_UPSTREAM_NAME) + + ) + ) + repo.update(client.post(REPO_PATH, gen_repo())) + sync(cfg, remote, repo) + finally: + if remote: + client.delete(remote['pulp_href']) + if repo: + client.delete(repo['pulp_href']) + return client.get(DOCKER_CONTENT_PATH)['results'] + + +skip_if = partial(selectors.skip_if, exc=SkipTest) +"""The ``@skip_if`` decorator, customized for unittest. + +:func:`pulp_smash.selectors.skip_if` is test runner agnostic. This function is +identical, except that ``exc`` has been set to ``unittest.SkipTest``. +""" diff --git a/extensions_admin/test/unit/__init__.py b/pulp_docker/tests/unit/__init__.py similarity index 100% rename from extensions_admin/test/unit/__init__.py rename to pulp_docker/tests/unit/__init__.py diff --git a/pulp_docker/tests/unit/test_models.py b/pulp_docker/tests/unit/test_models.py new file mode 100644 index 00000000..16e9754d --- /dev/null +++ b/pulp_docker/tests/unit/test_models.py @@ -0,0 +1,9 @@ +from django.test import TestCase + + +class TestNothing(TestCase): + """Test Nothing (placeholder).""" + + def test_nothing_at_all(self): + """Test that the tests are running and that's it.""" + self.assertTrue(True) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8490a40e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.towncrier] +package = "pulp_docker" +filename = "CHANGES.rst" +directory = "CHANGES/" +title_format = "{version} ({project_date})" +template = "CHANGES/.TEMPLATE.rst" +issue_format = "`#{issue} `_" diff --git a/rel-eng/packages/.readme b/rel-eng/packages/.readme deleted file mode 100644 index 8999c8db..00000000 --- a/rel-eng/packages/.readme +++ /dev/null @@ -1,3 +0,0 @@ -the rel-eng/packages directory contains metadata files -named after their packages. Each file has the latest tagged -version and the project's relative directory. diff --git a/rel-eng/packages/pulp-docker b/rel-eng/packages/pulp-docker deleted file mode 100644 index 29812a6e..00000000 --- a/rel-eng/packages/pulp-docker +++ /dev/null @@ -1 +0,0 @@ -0.2.1-0.2.beta ./ diff --git a/rel-eng/tito.props b/rel-eng/tito.props deleted file mode 100644 index fb74f3f7..00000000 --- a/rel-eng/tito.props +++ /dev/null @@ -1,5 +0,0 @@ -[globalconfig] -default_builder = tito.builder.Builder -default_tagger = tito.tagger.VersionTagger -changelog_do_not_remove_cherrypick = 0 -changelog_format = %s (%ae) diff --git a/run-tests.py b/run-tests.py deleted file mode 100755 index c27c0505..00000000 --- a/run-tests.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import subprocess - -from pulp.devel.test_runner import run_tests - -# Find and eradicate any existing .pyc files, so they do not eradicate us! -PROJECT_DIR = os.path.dirname(__file__) -subprocess.call(['find', PROJECT_DIR, '-name', '*.pyc', '-delete']) - -config_file = os.path.join(PROJECT_DIR, 'flake8.cfg') -subprocess.call(['flake8', '--config', config_file, PROJECT_DIR]) - -PACKAGES = [PROJECT_DIR, 'pulp_docker', ] - -TESTS = [ - 'common/test/unit/', - 'extensions_admin/test/unit/', -] - -PLUGIN_TESTS = ['plugins/test/unit/'] - -dir_safe_all_platforms = [os.path.join(os.path.dirname(__file__), x) for x in TESTS] -dir_safe_non_rhel5 = [os.path.join(os.path.dirname(__file__), x) for x in PLUGIN_TESTS] - -run_tests(PACKAGES, dir_safe_all_platforms, dir_safe_non_rhel5) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..f4595661 --- /dev/null +++ b/setup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +from setuptools import find_packages, setup + +requirements = [ + "pulpcore~=3.0rc7", + 'ecdsa~=0.13.2', + 'pyjwkest~=1.4.0', + 'pyjwt[crypto]~=1.7.1' +] + + +with open('README.rst') as f: + long_description = f.read() + +setup( + name='pulp-docker', + version='4.0.0b8.dev', + description='pulp-docker plugin for the Pulp Project', + long_description=long_description, + license='GPLv2+', + author='Pulp Team', + author_email='pulp-list@redhat.com', + url='http://pulpproject.org/', + python_requires='>=3.6', + install_requires=requirements, + include_package_data=True, + packages=find_packages(exclude=['tests', 'tests.*']), + classifiers=( + 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', + 'Operating System :: POSIX :: Linux', + 'Framework :: Django', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ), + entry_points={ + 'pulpcore.plugin': [ + 'pulp_docker = pulp_docker:default_app_config', + ] + } +) diff --git a/template_config.yml b/template_config.yml new file mode 100644 index 00000000..5d1d07a9 --- /dev/null +++ b/template_config.yml @@ -0,0 +1,48 @@ +# This config represents the latest values used when running the plugin-template. Any settings that +# were not present before running plugin-template have been added with their default values. + +additional_plugins: [] +all: false +black: false +bootstrap: false +check_commit_message: true +coverage: false +deploy_client_to_pypi: true +deploy_client_to_rubygems: true +deploy_daily_client_to_pypi: true +deploy_daily_client_to_rubygems: true +deploy_to_pypi: true +docs: false +docs_test: true +exclude_check_commit_message: false +exclude_coverage: true +exclude_deploy_client_to_pypi: false +exclude_deploy_client_to_rubygems: false +exclude_deploy_daily_client_to_pypi: false +exclude_deploy_daily_client_to_rubygems: false +exclude_deploy_to_pypi: false +exclude_docs_test: false +exclude_mariadb_test: false +exclude_test_bindings: false +plugin_app_label: docker +plugin_camel: PulpDocker +plugin_camel_short: Docker +plugin_caps: PULP_DOCKER +plugin_caps_short: DOCKER +plugin_dash: pulp-docker +plugin_dash_short: docker +plugin_name: pulp_docker +plugin_snake: pulp_docker +plugin_snake_short: docker +pulp_settings: + token_server: $(hostname):24816/token + token_signature_algorithm: ES256 +pydocstyle: true +pypi_username: pulp +test: false +test_bindings: true +test_performance: false +travis: true +travis_notifications: None +verbose: false + diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..2544df3c --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,3 @@ +# All test requirements +-r functest_requirements.txt +-r unittest_requirements.txt diff --git a/unittest_requirements.txt b/unittest_requirements.txt new file mode 100644 index 00000000..932a8957 --- /dev/null +++ b/unittest_requirements.txt @@ -0,0 +1 @@ +mock