From b383d4a80edceaf446d719454b89b5d4584216a4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 25 Aug 2018 09:48:56 +0900 Subject: Update job content syntax and add more examples --- src/zuulv3/zuul.rst | 427 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 335 insertions(+), 92 deletions(-) diff --git a/src/zuulv3/zuul.rst b/src/zuulv3/zuul.rst index b38675b..efd6899 100644 --- a/src/zuulv3/zuul.rst +++ b/src/zuulv3/zuul.rst @@ -4,7 +4,7 @@ .. pygments yaml? (only file breaks (---) tinted) .. slide on high level v3 changes .. slide on nodepool - + .. transition:: dissolve :duration: 0.4 @@ -32,7 +32,7 @@ Red Hat i work for .. ansi:: images/redhat.ans - + OpenStack ========= .. hidetitle:: @@ -72,7 +72,7 @@ Presentation Checklist Spoilers ======== -* What Zuul v3 does +* What Zuul does * multiple repositories * integrated deliverable @@ -128,6 +128,17 @@ Large numbers of * Changes * Code Repositories (1955 as of this morning) +Not Bragging About Scale +======================== + +OpenStack Scale Comparison +========================== + + * 2KJPH (2,000 jobs per hour) + * Build Nodes from 13 Regions of 5 Public and 2 Private OpenStack Clouds + * Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro, Limestone + * 10,000 changes merged per month + OpenStack Scale Comparison ========================== @@ -136,6 +147,9 @@ OpenStack Scale Comparison * Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro, Limestone * 10,000 changes merged per month + * By comparison, our friends at the amazing project Ansible received + 13,000 changes and had merged 8,000 of them in its first 4 years. + Four Opens ========== @@ -203,7 +217,7 @@ Gerrit but zuul is doing a lot of work behind the scenes, and if you look closer, this is what you see - + .. ansi:: images/color-gertty.ans Zuul Architecture @@ -227,11 +241,18 @@ Nodepool * Creates and destroys (at least) a vm for every job (Remember that 2,000 jobs per hour number?) - + +Zuul is not New +=============== + + * Has been in Production for OpenStack for Six Years + * Zuul v3 first release where not-OpenStack is first-class use case + * Zuul is now a top-level effort of OpenStack Foundation + Not just for OpenStack ====================== - * Zuul v3 is in production for OpenStack (in OpenStack VMs) + * Zuul is in production for OpenStack (in OpenStack VMs) Also running at: @@ -254,6 +275,9 @@ Zuul in a nutshell * Collects/reports results * Potentially merges change +All in Service of Gating +======================== + Gating ====== @@ -279,7 +303,7 @@ Presentation Checklist ====================== :: - + [x] Logos [x] Architecture diagram [x] Cows @@ -297,7 +321,7 @@ Zuul Simulation =============== .. transition:: cut .. container:: handout - + * todo .. ansi:: images/zsim-01.ans @@ -532,11 +556,11 @@ Live Configuration Changes source: gerrit: config-repos: - - 'project-config' + - openstack-infra/project-config project-repos: - - 'nova' - - 'keystone' - - 'devstack-gate' + - openstack/nova + - openstack/keystone + - openstack-infra/devstack-gate Zuul Startup ============ @@ -571,9 +595,9 @@ When .zuul.yaml Changes * Asks mergers for updated content * Splices into configuration used for that change * Works with cross-repo dependencies - + ("This change depends on a change to the job definition") - + How do you use this thing? ========================== .. transition:: tilt @@ -586,7 +610,7 @@ Pipelines * A process definition that connects git repositories, jobs, and reporting mechanisms. * A context to fix a set of jobs to each project. - + Check Pipeline ============== @@ -599,7 +623,7 @@ Check Pipeline trigger: gerrit: - event: patchset-created - - event: change-restored + - event: change-restored success: gerrit: verified: 1 @@ -638,23 +662,64 @@ Job .. code:: yaml - job: - name: 'base' - timeout: '30m' - nodes: 'ubuntu-xenial' - workspace: '/opt/workspace' - pre-run: - - 'setup-host' + name: base + parent: null + description: | + The base job for Zuul. + timeout: 1800 + nodeset: + nodes: + - name: primary + label: centos-7 + pre-run: playbooks/base/pre.yaml post-run: - - 'archive-logs' + - playbooks/base/post-ssh.yaml + - playbooks/base/post-logs.yaml + secrets: + - site_logs Simple Job ========== +.. code:: yaml + + - job: + name: tox + pre-run: playbooks/setup-tox.yaml + run: playbooks/tox.yaml + post-run: playbooks/fetch-tox-output.yaml + +Simple Job Inheritance +====================== + .. code:: yaml - job: - name: 'python27' - parent: 'base' + name: tox-py36 + parent: tox + vars: + tox_envlist: py36 + +Inheritance Works Like An Onion +=============================== + + * pre-run playbooks run in order of inheritance + * run playbook of job runs + * post-run playbooks run in reverse order of inheritance + * If pre-run playbooks fail, job is re-tried + * All post-run playbooks run - as far as pre-run playbooks got + +Inheritance Example +=================== + +For tox-py36 job + + * base pre-run playbooks/base/pre.yaml + * tox pre-run playbooks/setup-tox.yaml + * tox run playbooks/tox.yaml + * tox post-run playbooks/fetch-tox-output.yaml + * base post-run playbooks/base/post-ssh.yaml + * base post-run playbooks/base/post-logs.yaml Simple Job Variant ================== @@ -662,45 +727,102 @@ Simple Job Variant .. code:: yaml - job: - name: 'python27' - branch: 'stable/mitaka' - nodes: 'ubuntu-trusty' - + name: tox-py27 + branches: stable/mitaka + nodeset: + - name: ubuntu-trusty + label: ubuntu-trusty + +Nodesets for Multi-node Jobs +============================ + +.. code:: yaml + + - nodeset: + name: ceph-cluster + nodes: + - name: controller + label: centos-7 + - name: compute1 + label: fedora-28 + - name: compute2 + label: fedora-28 + groups: + - name: ceph-osd + nodes: + - controller + - name: ceph-monitor + nodes: + - controller + - compute1 + - compute2 + Multi-node Job ============== -.. container:: handout - nodepool, shrews +* nodesets are provided to Ansible for jobs in inventory .. code:: yaml - job: - name: 'devstack-multinode' - parent: 'base' - nodes: - - name: 'controller' - image: 'ubuntu-xenial' - - name: 'compute' - image: 'ubuntu-xenial' + name: ceph-multinode + nodeset: ceph-cluster + run: playbooks/install-ceph.yaml + +Multi-node Ceph Job Content +=========================== + +.. code:: yaml + + - hosts: all + roles: + - install-ceph + + - hosts: ceph-osd + roles: + - start-ceph-osd + + - hosts: ceph-monitor + roles: + - start-ceph-monitor + + - hosts: all + roles: + - do-something-interesting Projects ======== * Projects are git repositories * Specify a set of jobs for each pipeline - -Project -======= +* golang git repo naming as been adopted: + +:: + + zuul@ubuntu-xenial:~$ find /home/zuul/src -mindepth 3 -maxdepth 3 -type d + /home/zuul/src/git.openstack.org/openstack-infra/shade + /home/zuul/src/git.openstack.org/openstack/keystoneauth + /home/zuul/src/git.openstack.org/openstack/os-client-config + /home/zuul/src/github.com/ansible/ansible + +Project Config +============== + + * Specify a set of jobs for each pipeline .. code:: yaml - project: - name: 'nova' check: jobs: - - python27 - - python35 - - docs + - openstack-tox-py27 + - openstack-tox-py35 + - openstack-tox-docs + gate: + jobs: + - openstack-tox-py27 + - openstack-tox-py35 + - openstack-tox-docs Project with Local Variant ========================== @@ -708,14 +830,18 @@ Project with Local Variant .. code:: yaml - project: - name: 'nova' check: jobs: - - python27 - - python35 - - docs - - pypy: - voting: false + - openstack-tox-py27 + - openstack-tox-py35 + - openstack-tox-py36: + voting: false + - openstack-tox-docs + gate: + jobs: + - openstack-tox-py27 + - openstack-tox-py35 + - openstack-tox-docs Project with More Local Variants ================================ @@ -723,15 +849,14 @@ Project with More Local Variants .. code:: yaml - project: - name: 'nova' check: jobs: - - python27 - - python35 - - docs: - files: '^docs/.*$' - - pypy: - voting: false + - openstack-tox-py27 + - openstack-tox-py35 + - openstack-tox-py36: + voting: false + - openstack-tox-docs: + files: '^docs/.*$' Project with Many Local Variants ================================ @@ -739,34 +864,58 @@ Project with Many Local Variants .. code:: yaml - project: - name: 'nova' check: jobs: - - python27: - nodes: 'ubuntu-xenial' - - python27: - branch: 'stable/newton' - nodes: 'ubuntu-trusty' - - python35 - - docs: - files: '^docs/.*$' - - pypy: - voting: false + - openstack-tox-py27: + nodeset: + - name: centos-7 + label: centos-7 + - openstack-tox-py27: + branches: stable/newton + nodeset: + - name: ubuntu-trusty + label: ubuntu-trusty + - openstack-tox-py35 + - openstack-tox-py36: + voting: false + - openstack-tox-docs: + files: '^docs/.*$' + +Project With Central and Local Config +===================================== + +.. code:: yaml + + # In git.openstack.org/openstack-infra/project-config: + - project: + name: openstack/nova + templates: + - openstack-tox-jobs + +.. code:: yaml + + # In git.openstack.org/openstack/nova/.zuul.yaml: + - project: + check: + - nova-placement-functional-devstack Project with Job Dependencies ============================= .. code:: yaml - - project: - name: nova - release: - jobs: - - build-tarball: - jobs: - - upload-tarball: - jobs: - - update-mirror + - project: + release: + jobs: + - build-artifacts + - upload-tarball: + dependencies: build-artifacts + - upload-pypi: + dependencies: build-artifacts + - notify-mirror: + dependencies: + - upload-tarball + - upload-pypi Playbooks ========= @@ -774,32 +923,37 @@ Playbooks * Jobs run playbooks * Playbooks may be defined centrally or in the repo being tested * Playbooks can use roles from current or other Zuul repos or Galaxy +* Playbooks are not allowed to execute content on 'localhost' -Devstack-gate / Tempest Playbook -================================ +devstack-tempest Run Playbook +============================= .. code:: yaml - # devstack-gate / tempest playbook - --- - hosts: all - roles: - - setup-multinode-networking - - partition-swap - - configure-mirrors - - run-devstack - - run-tempest + # Changes that run through devstack-tempest are likely to have an impact on + # the devstack part of the job, so we keep devstack in the main play to + # avoid zuul retrying on legitimate failures. + - hosts: all + roles: + - run-devstack + + # We run tests only on one node, regardless how many nodes are in the system + - hosts: tempest + roles: + - setup-tempest-run-dir + - setup-tempest-data-dir + - acl-devstack-files + - run-tempest Simple Shell Playbook ===================== .. code:: yaml - --- hosts: controller roles: - shell: | - cd $WORKSPACE + cd {{ zuul.project.src_dir }} ./run_tests.sh @@ -808,7 +962,96 @@ Test Like Production If you use Ansible for deployment, your test and deployment processes and playbooks are the same - + +What if you don't use Ansible? +============================== + +OpenStack Infra Control Plane uses Puppet (for now) +=================================================== + +.. code:: yaml + + # In git.openstack.org/openstack-infra/project-config/roles/legacy-install-afs-with-puppet/tasks/main.yaml + - name: Install puppet + shell: ./install_puppet.sh + args: + chdir: "{{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/system-config" + environment: + # Skip setting up pip, our images have already done this. + SETUP_PIP: "false" + become: yes + + - name: Copy manifest + copy: + src: manifest.pp + dest: "{{ ansible_user_dir }}/manifest.pp" + + - name: Run puppet + puppet: + manifest: "{{ ansible_user_dir }}/manifest.pp" + become: yes + +Secrets +======= + +* Inspired by Kubernetes Secrets API +* Projects can add named encrypted secrets to their .zuul.yaml file +* Jobs can request to use secrets by name +* Jobs using secrets are not reconfigured speculatively +* Secrets can only be used by the same project they are defined in +* Public key per project: + ``{{ zuul_url }}/{{ tenant }}/{{ project }}.pub`` + +:: + GET https://zuul.openstack.org/openstack-infra/shade.pub + +Secret Example (note, no admins had to enable this) +=================================================== + +.. code:: yaml + + # In git.openstack.org/openstack/loci/.zuul.yaml: + - secret: + name: loci_docker_login + data: + user: loci-username + password: !encrypted/pkcs1-oaep + - gUEX4eY3JAk/Xt7Evmf/hF7xr6HpNRXTibZjrKTbmI4QYHlzEBrBbHey27Pt/eYvKKeKw + hk8MDQ4rNX7ZK1v+CKTilUfOf4AkKYbe6JFDd4z+zIZ2PAA7ZedO5FY/OnqrG7nhLvQHE + 5nQrYwmxRp4O8eU5qG1dSrM9X+bzri8UnsI7URjqmEsIvlUqtybQKB9qQXT4d6mOeaKGE + 5h6Ydkb9Zdi4Qh+GpCGDYwHZKu1mBgVK5M1G6NFMy1DYz+4NJNkTRe9J+0TmWhQ/KZSqo + 4ck0x7Tb0Nr7hQzV8SxlwkaCTLDzvbiqmsJPLmzXY2jry6QsaRCpthS01vnj47itoZ/7p + taH9CoJ0Gl7AkaxsrDSVjWSjatTQpsy1ub2fuzWHH4ASJFCiu83Lb2xwYts++r8ZSn+mA + hbEs0GzPI6dIWg0u7aUsRWMOB4A+6t2IOJibVYwmwkG8TjHRXxVCLH5sY+i3MR+NicR9T + IZFdY/AyH6vt5uHLQDU35+5n91pUG3F2lyiY5aeMOvBL05p27GTMuixR5ZoHcvSoHHtCq + 7Wnk21iHqmv/UnEzqUfXZOque9YP386RBWkshrHd0x3OHUfBK/WrpivxvIGBzGwMr2qAj + /AhJsfDXKBBbhGOGk1u5oBLjeC4SRnAcIVh1+RWzR4/cAhOuy2EcbzxaGb6VTM= + +Secret Example +============== + +.. code:: yaml + + # In git.openstack.org/openstack/loci/.zuul.yaml: + - job: + name: publish-loci-cinder + parent: loci-cinder + post-run: playbooks/push + secrets: + - loci_docker_login + + # In git.openstack.org/openstack/loci/playbooks/push.yaml: + - hosts: all + tasks: + - include_vars: vars.yaml + + - name: Push project to DockerHub + block: + - command: docker login -u {{ loci_docker_login.user }} -p {{ loci_docker_login.password }} + no_log: True + - command: docker push openstackloci/{{ project }}:{{ branch }}-{{ item.name }} + with_items: "{{ distros }}" + Important Links =============== -- cgit v1.2.3