From 1f40de78d19e97f2b52446becbddb59a77516d15 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 23 Aug 2018 21:53:02 -0500 Subject: Import jim's changes to zuulv3 talk --- src/zuulv3/images/architecture.ans | 38 +- src/zuulv3/images/orig/title.ans | 40 +- src/zuulv3/images/title.ans | 44 +- src/zuulv3/images/zuul.ans | 31 +- src/zuulv3/start.sh | 6 +- src/zuulv3/zuul.rst | 948 +++++++++---------------------------- 6 files changed, 298 insertions(+), 809 deletions(-) (limited to 'src') diff --git a/src/zuulv3/images/architecture.ans b/src/zuulv3/images/architecture.ans index 0698ccc..ad308eb 100644 --- a/src/zuulv3/images/architecture.ans +++ b/src/zuulv3/images/architecture.ans @@ -4,42 +4,42 @@   - ┌─────────────┐ + ┌─────────────┐  - ┌───────┤ Zuul Merger │ + ┌───────┤ Zuul Merger │     - │ └──────┬──────┘ + ┌────┴───┐ └──┬───┬──────┘  - │ │ + ┌──┤ Gerrit ├──┐ │ │  - │ │ + │ └────────┘ │ ┌─┘ │  -  ┌────┴───┐ ┌──┴───┐ ┌───────────────┐ +   │ ┌────────┐ │ │ ┌──┴───┐ ┌───────────────┐  - ┌───────┤ Gerrit ├───────┤ Zuul ├─────┤ Zuul Executor │ + ┌─────┼──┤  Web  ├──┼────┤ Zuul ├─────┤ Zuul Executor │   - ____ └────────┘ └──┬───┘ │ ┌───────┐ │ + ____ │ └────┬───┘ │ │ └──┬───┘ │ ┌───────┐ │  - |... | │ │  │Ansible│ │  + |... | │ ┌────┴───┐ │ │ │ │  │Ansible│ │  - |... | │ └───┴───┬───┴───┘ + |... | ├──┤ GitHub ├──┼─┘ │ └───┴───┬───┴───┘  -/______\ ┌────┴─────┐ │ +/______\ │ └────────┘ │ ┌────┴─────┐ │  - o │ Nodepool │ │ + o │ ┌────────┐ │ │ Nodepool │ │  - -|-   └────┬─────┘ │ + -|- └──┤ Finger ├──┘ └────┬─────┘ │  - / \ │ │ + / \ └────────┘ │ │  - Dev Cloud │  │ + Dev Cloud │  │  -  Node 1 ─────────────┤ +  Node 1 ─────────────┤  -  ...  │ +  ...  │  -  Node 2 ─────────────┘ +  Node 2 ─────────────┘  -(Not to scale)   +(Not to scale)     \ No newline at end of file diff --git a/src/zuulv3/images/orig/title.ans b/src/zuulv3/images/orig/title.ans index 82ef305..8442c58 100644 --- a/src/zuulv3/images/orig/title.ans +++ b/src/zuulv3/images/orig/title.ans @@ -1,42 +1,40 @@ - ______________ .. .. ______________ +͵  -( (  \ \_____)____(_____/ /  ) ) + Ĵ  - \ ) `'   `' ( / +   - )/   \( - - /'  REST APIs and the Return of the  `\  +  -O   ___ ___ _ _ ___ ___ _ ___   O +  -  / __/ _ \| \| / __|/ _ \| | | __|  +  - | (_| (_) | .` \__ \ (_) | |__| _|   +  _____ _  -  \___\___/|_|\_|___/\___/|____|___|  + |__ / _ _ _| |  -  + / / | | | | | | |  - Application  + / /| |_| | |_| | |  -  + /____\__,_|\__,_|_|  -O  James E. Blair   O +  - \. ./  +  - )\  ,____ ____,  /( +  - / ) / ____\ ____ /____ \ ( \ +  -( (_________/_/ ) ( \_\_________) ) +  - ~-'  `' `'  `-~  +   - + Ĵ  - +͵   diff --git a/src/zuulv3/images/title.ans b/src/zuulv3/images/title.ans index c784327..8857086 100644 --- a/src/zuulv3/images/title.ans +++ b/src/zuulv3/images/title.ans @@ -1,45 +1,47 @@ - ______________ .──. .──. ______________ +══════════════════════╡│││╞═════════════════════  -( (  \ \_____)____(_____/ /  ) ) + ────────────────────┤│├───────────────────  - \ ) `─────'   `─────' ( / + ──────────────────│─────────────────  - )/   \( - - /'  REST APIs and the Return of the  `\  +  -O   ___ ___ _ _ ___ ___ _ ___   O +  -│  / __/ _ \| \| / __|/ _ \| | | __| │ +  -│ | (_| (_) | .` \__ \ (_) | |__| _|  │ +  _____ _  -│  \___\___/|_|\_|___/\___/|____|___| │ + |__ / _ _ _| |  -│ │ + / / | | | | | | |  -│ Application │ + / /| |_| | |_| | |  -│ │ + /____\__,_|\__,_|_|  -O  James E. Blair   O +  - \. ./  +  - )\  ,____ ____,  /( + Monty Taylor  - / ) / ____\ ____ /____ \ ( \ + irc: mordred  -( (_________/_/ ) ( \_\_________) ) + twitter: @e_monty  - ~-'  `──' `──'  `-~  + Red Hat   - + ──────────────────│───────────────── + + ────────────────────┤│├─────────────────── + +══════════════════════╡│││╞═════════════════════    - \ No newline at end of file + diff --git a/src/zuulv3/images/zuul.ans b/src/zuulv3/images/zuul.ans index 6369083..44094b8 100644 --- a/src/zuulv3/images/zuul.ans +++ b/src/zuulv3/images/zuul.ans @@ -9,28 +9,25 @@  - -  _____ _ - - |__ / _ _ _| | - - / / | | | | | | | - - / /| |_| | |_| | | - - /____\__,_|\__,_|_| - - - - + + ╱╲ + ╱ ╲ + ──────────────── + ╲┌─┬────────┬─┐╱ + ├─┼────────┼─┤ + ╱│ │ ▏ │ │╲ + ╱ │ │ ▏ │ │ ╲ + ╱ │ │ ▏ │ │ ╲ + ╱ │ │ ▏ │ │ ╲ + ────┴─┴────────┴─┴──── + + Z U U L    - - - ──────────────────│───────────────── + ──────────────────│─────────────────  ────────────────────┤│├───────────────────  diff --git a/src/zuulv3/start.sh b/src/zuulv3/start.sh index 9eaf8b6..31b2cee 100755 --- a/src/zuulv3/start.sh +++ b/src/zuulv3/start.sh @@ -1,8 +1,8 @@ #!/bin/sh -mate-terminal --geometry 68x24 -x presentty $(pwd)/zuul.rst & -#geeqie -t images & +gnome-terminal --geometry 68x24 -- ~/presentty/venv/bin/presentty zuul.rst & +geeqie -t images & sleep 1 -mate-terminal --maximize -x presentty-console $(pwd)/zuul.rst & +gnome-terminal --maximize -- ~/presentty/venv/bin/presentty-console zuul.rst & diff --git a/src/zuulv3/zuul.rst b/src/zuulv3/zuul.rst index 3ce2437..b38675b 100644 --- a/src/zuulv3/zuul.rst +++ b/src/zuulv3/zuul.rst @@ -4,7 +4,7 @@ .. pygments yaml? (only file breaks (---) tinted) .. slide on high level v3 changes .. slide on nodepool - + .. transition:: dissolve :duration: 0.4 @@ -23,43 +23,23 @@ Preshow Zuul ==== .. hidetitle:: -.. ansi:: images/zuul.ans - -Meta Information -================ - -* This Talk is Free Software: http://git.inaugust.com/cgit/inaugust.com/tree/src/zuulv3 -* Twitter: @e_monty -* Freenode: mordred -* email: mordred@inaugust.com +.. ansi:: images/title.ans Red Hat ======= - .. hidetitle:: .. container:: handout - - * I work for Red Hat in the Office of Technology as the Chief Architect - for CI/CD + i work for .. ansi:: images/redhat.ans - + OpenStack ========= -.. container:: handout - - * I work on OpenStack. - * I sit on the Technical Committee. I was on the Board of Directors - .. hidetitle:: .. ansi:: images/openstack.ans OpenStack Infra =============== -.. container:: handout - - * My primary technical role with OpenStack is working on the OpenStack CI - system. :: @@ -73,98 +53,125 @@ OpenStack Infra Zuul ==== -.. container:: handout - - * As part of working on OpenStack Infra I work on Zuul - .. hidetitle:: .. ansi:: images/zuul.ans Ansible ======= - -.. container:: handout - - * And as part of working on Zuul and on OpenStack I work on Ansible - * I maintain the OpenStack modules for Ansible as well as the shade library - .. hidetitle:: .. ansi:: images/ansible.ans Presentation Checklist ====================== -.. container:: handout - - * Every good presentation needs logos, so we're starting well - :: [X] Logos - Spoilers ======== -* What the old version of Zuul (v2) was +* What Zuul v3 does - * a nifty project gating system + * multiple repositories + * integrated deliverable + * gated commits + * open tooling + * nobody is special + * testing like deployment -* What the new version of Zuul (v3) is +OpenStack Is +============ - * multinode support - * live configuration changes - * better job definition - * sharable job definition - * testing like deployment + * Federated + * Distributed + * Large + * Open + * Not Alone -What do I mean by Massive Scale? -================================ +Federated +========= - * Contributors (~2k / 6 month period) - * Companies - * Changes - * Code Repositories (1868 as of this morning) - * Communities + * Hundreds of involved companies + * No 'main' company + * "Decisions are made by those who show up" + * Union of priorities/use cases -OpenStack Scale Comparison +Impact of being Federated +========================= + + * No company can appoint people to positions in the project + * The project cannot fire anyone + * Variable background of contributors + * Heavy reliance on consensus + +Distributed +=========== + + * There is no office + * Contributor base is global + * Multitude of contributor backgrounds + +Impact of being Distributed =========================== - * 2KJPH (2,000 jobs per hour) - * Nodes from 12 Regions of 5 Public and 1 Private OpenStack Clouds - (Thanks Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro) - * 10,000 changes merged per month + * Tooling must empower all contributors, regardless of background, + skill level or cultural context + * Heavy preference for text-based communication + * Cannot assume US-centric needs or solutions + +Large numbers of +================ + + * Contributors (\~2k in any given 6 month period) + * Changes + * Code Repositories (1955 as of this morning) OpenStack Scale Comparison ========================== * 2KJPH (2,000 jobs per hour) - * Nodes from 12 Regions of 5 Public and 1 Private OpenStack Clouds - (Thanks Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro) + * Build Nodes from 13 Regions of 5 Public and 2 Private OpenStack Clouds + * Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro, Limestone * 10,000 changes merged per month - * By comparison, our friends at the amazing project Ansible received - 13,000 changes and had merged 8,000 of them in its first 4 years. +Four Opens +========== -Pretty Things to Look for Scale -=============================== + * Open Source + (we don't hold back Enterprise features, we don't cripple things) + * Open Design + (design process open to all, decisions are not made inside company doors) + * Open Development + (public source code, public code review, all code is reviewed and gated) + * Open Community + (lazy consensus, democratic leadership from participants, + public logged meetings in IRC, public archived mailing lists) + +We're Not Alone +=============== - * http://grafana.openstack.org/dashboard/db/zuul-status - * http://grafana.openstack.org/dashboard/db/nodepool - * http://zuul.openstack.org/ + * Dependencies (libvirt/kvm/xen, mysql/pg, rabbit, + python/javascript, ceph/gluster, ansible/salt/puppet/chef, ovs/odl) + * Adjacencies (kubernetes, ansible, terraform, opnfv, spinnaker) + * Vendors (plugins, products, services, distros) -Dealing With Scale -================== +Developer Process In a Nutshell +=============================== - * Egalitarian Process - * Balance Centralized vs Distributed - * Code Review plus Enforced Testing + * Code Review - nobody has direct commit/push access + * 3rd-Party CI for vendors + * Gated Commits OpenStack Developer Workflow ============================ .. container:: handout + * Who has submitted a patch? + * Who wants to? + * (Who is here because the name of this talk is weird?) + :: Hack Review Test @@ -187,39 +194,18 @@ Gerrit .. hidetitle:: .. container:: handout - the primary interface for our developers is the code review system - gerrit. No matter how complex zuul becomes, this is still primary - focus we want the developers to have. - explain patch upload, zuul runs, test results displayed in gerrit this is all the interface to zuul users need to see + switch to actual gertty screenshot + + also show zuul status page + but zuul is doing a lot of work behind the scenes, and if you look closer, this is what you see - + .. ansi:: images/color-gertty.ans -Github Developer Workflow -========================= -.. container:: handout - -:: - - Hack Review Test - ========= ========== ========== - - push approve - +-------------+ +-------------+ - | | | | - +------+--+ +--v----+--+ +--v-------+ - | | | | | | - | $EDITOR | | Github | | Zuul | - | | | | | | - +------^--+ +--+----^--+ +--+-------+ - | | | | - +-------------+ +-------------+ - clone merge - Zuul Architecture ================= @@ -236,28 +222,37 @@ Presentation Checklist Nodepool ======== -.. container:: handout +* A separate program that works very closely with *Zuul* +* Builds images daily and uploads to clouds +* Creates and destroys (at least) a vm for every job - nodepool builds nodes for zuul - Remember that 2,000 jobs per hour number? - Each job gets a fresh VM - that's 2,000 VMs per hours - Treats our 12 regions across 6 clouds as one REALLY big cloud + (Remember that 2,000 jobs per hour number?) + +Not just for OpenStack +====================== -:: + * Zuul v3 is in production for OpenStack (in OpenStack VMs) - * A separate program that works very closely with *zuul* - * Builds images daily and uploads to clouds - * Creates and destroys (at least) a VM for every job - * Supports using pre-existing nodes (static provider) + Also running at: -Nodepool can use pre-existing images, BUT ... -============================================= + * BMW (control plane in OpenShift) + * Easystack + * GoDaddy (control plane in Kubernetes) + * OpenContrail + * OpenLab + * Red Hat + * others ... - * Clouds have 'helpful' differences between base images - * Cloud images have 'helpful' software pre-installed - * Distros have 'helpful' different user names - * Most clouds use DHCP for networking, but some don't - * We can add pre-cached content +Zuul in a nutshell +================== + + * Listens for code events + * Prepares appropriate job config and git repo states + * Allocates nodes for test jobs + * Pushes git repo states to nodes + * Runs user-defined Ansible playbooks + * Collects/reports results + * Potentially merges change Gating ====== @@ -269,22 +264,22 @@ Co-gating ========= .. cowsay:: Changes to a set of repositories merge monotonically such - that each change is tested with the current state of all - other related repositories before it merges. + that each change is tested with the current state of all + the other related repositories before it merges. Parallel Co-gating ================== .. cowsay:: Changes are serialized such that each change is tested - with all of the changes ahead of it to satisfy the - gating requirement while being able to run tests for - multiple changes simultaneously. + with all of the changes ahead of it to satisfy the + co-gating requirement while being able to run tests for + multiple changes simultaneously. Presentation Checklist ====================== :: - + [x] Logos [x] Architecture diagram [x] Cows @@ -294,9 +289,7 @@ Zuul Simulation .. transition:: pan .. container:: handout - * That was a lot of words - let's walk through it one step at a time - * Here we have two git repos, called nova and keystone, and their - current HEAD state + * todo .. ansi:: images/zsim-00.ans @@ -304,8 +297,8 @@ Zuul Simulation =============== .. transition:: cut .. container:: handout - - * A change is approved for Nova + + * todo .. ansi:: images/zsim-01.ans @@ -314,9 +307,7 @@ Zuul Simulation .. transition:: cut .. container:: handout - * Zuul starts running jobs for it - * The tests will test the current state of nova and keystone PLUS this nova - change + * todo .. ansi:: images/zsim-02.ans @@ -325,7 +316,7 @@ Zuul Simulation .. transition:: cut .. container:: handout - * A change is approved for Keystone + * todo .. ansi:: images/zsim-03.ans @@ -334,8 +325,7 @@ Zuul Simulation .. transition:: cut .. container:: handout - * The tests will test the current state of nova and keystone PLUS this nova - change + * todo .. ansi:: images/zsim-04.ans @@ -501,159 +491,32 @@ Zuul Simulation .. ansi:: images/zsim-22.ans -Cross-Project Problem -===================== - - * User reports bug in shade - auto_ip is not discovering their NAT properly - * Two fixes, one to detection algorithm, one to config override - * Config override requires adding support to os-client-config - * Once support is added to os-client-config, it can be consumed in shade - * How do we integration test this without releasing os-client-config? - Cross-Project Dependencies ========================== -Testing or gating dependencies (including jobs) manually specified by -developers +Testing or gating dependencies manually specified by developers .. container:: progressive - * shade https://review.openstack.org/#/c/513913/ + * shade https://review.openstack.org/513913 Add unittest tips jobs - - Change-ID: I5b411be5c5aa43535fa89a51d6099aadd7a8ea60 - * os-client-config https://review.openstack.org/#/c/513915 + * os-client-config https://review.openstack.org/513915 Add shade-tox-tips jobs - Change-ID: Ie3e9a4deca1d74b94e810e87e130706fe15fe2c9 - - Depends-On: I5b411be5c5aa43535fa89a51d6099aadd7a8ea60 - * os-client-config https://review.openstack.org/#/c/513751/ + Depends-On: https://review.openstack.org/513913 + * os-client-config https://review.openstack.org/513751 Added nat_source flag for networks - Change-ID: I3d8dd6d734a1013d2d4a43e11c3538c3a345820b - - * shade https://review.openstack.org/#/c/513914 + Depends-On: https://review.openstack.org/513915 + * shade https://review.openstack.org/51391 Add support for configured NAT source variable - Change-Id: I4b50c2323a487b5ce90f9d38a48be249cfb739c5 - - Depends-On: I3d8dd6d734a1013d2d4a43e11c3538c3a345820b - -shade: Add unittest tips jobs -============================= - -* In git.openstack.org/openstack-infra/shade/.zuul.yaml: - -.. code:: yaml - - - job: - name: shade-tox-py27-tips - parent: openstack-tox-py27 - description: | - Run tox python 27 unittests against master of important libs - required-projects: - - openstack-infra/shade - - openstack/keystoneauth - - openstack/os-client-config - - - job: - name: shade-tox-py35-tips - parent: openstack-tox-py35 - description: | - Run tox python 35 unittests against master of important libs - required-projects: - - openstack-infra/shade - - openstack/keystoneauth - - openstack/os-client-config - -shade: Add unittest tips project-template -========================================= - -* In git.openstack.org/openstack-infra/shade/.zuul.yaml: - -.. code:: yaml - - - project-template: - name: shade-tox-tips - check: - jobs: - - shade-tox-py27-tips - - shade-tox-py35-tips - gate: - jobs: - - shade-tox-py27-tips - - shade-tox-py35-tips - -shade: Add unittest tips project-template to project -==================================================== - -* In git.openstack.org/openstack-infra/shade/.zuul.yaml: - -.. code:: yaml - - - project: - templates: - - publish-to-pypi - - publish-openstack-sphinx-docs - - shade-tox-tips - -os-client-config: Add shade-tox-tips jobs -========================================= - -* In git.openstack.org/openstack/os-client-config/.zuul.yaml: - -.. code:: yaml + Depends-On: https://review.openstack.org/513751 - - project: - templates: - - shade-tox-tips - check: - jobs: - - legacy-osc-dsvm-functional-tips: - voting: false - -os-client-config: Add nat_source flag for networks -================================================== - -:: - - diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py - index 2e97629..d1a6983 100644 - --- a/os_client_config/cloud_config.py - +++ b/os_client_config/cloud_config.py - @@ -581,3 +581,10 @@ class CloudConfig(object): - if net['nat_destination']: - return net['name'] - return None - + - + def get_nat_source(self): - + """Get network used for NAT source.""" - + for net in self.config['networks']: - + if net.get('nat_source'): - + return net['name'] - + return None - -shade: Add support for configured NAT source variable -===================================================== - -:: - - Zuul 10-21 13:57 - Patch Set 5: Verified-1 - Build failed. - openstack-tox-pep8 SUCCESS in 2m 29s - openstack-tox-py27 FAILURE in 2m 34s - build-openstack-releasenotes SUCCESS in 2m 47s - openstack-tox-py35 FAILURE in 2m 41s - openstack-tox-cover POST_FAILURE in 3m 52s (non-voting) - build-openstack-sphinx-docs SUCCESS in 2m 57s - shade-tox-py27-tips SUCCESS in 3m 18s - shade-tox-py35-tips SUCCESS in 2m 28s Live Configuration Changes ========================== @@ -668,16 +531,12 @@ Live Configuration Changes name: openstack source: gerrit: - config-projects: - - project-config - untrusted-projects: - - openstack-infra/zuul-jobs: - shadow: openstack-infra/project-config - - openstack-infra/openstack-zuul-jobs - - openstack-infra/nodepool - - openstack-infra/shade - - openstack-infra/zuul - - openstack/requirements + config-repos: + - 'project-config' + project-repos: + - 'nova' + - 'keystone' + - 'devstack-gate' Zuul Startup ============ @@ -697,10 +556,9 @@ Zuul Startup * Read config file * Ask mergers for branches of each repo -* Ask mergers for .zuul.yaml file for each branch of each repo +* Ask mergers for .zuul.yaml for each branch -``.zuul.yaml`` can be ``^\.?zuul.yaml$`` file or ``^\.?zuul.d$`` run-parts -directory. + of each repo .. ansi:: images/startup2.ans @@ -713,9 +571,9 @@ When .zuul.yaml Changes * Asks mergers for updated content * Splices into configuration used for that change * Works with cross-repo dependencies - + ("This change depends on a change to the job definition") - + How do you use this thing? ========================== .. transition:: tilt @@ -725,11 +583,10 @@ How do you use this thing? Pipelines ========= -* Describes the process flow and lifecycle **for a change** * A process definition that connects git repositories, jobs, and reporting mechanisms. * A context to fix a set of jobs to each project. - + Check Pipeline ============== @@ -742,7 +599,7 @@ Check Pipeline trigger: gerrit: - event: patchset-created - - event: change-restored + - event: change-restored success: gerrit: verified: 1 @@ -755,6 +612,7 @@ Gate Pipeline - pipeline: name: gate manager: dependent + source: gerrit trigger: gerrit: - event: comment-added @@ -765,153 +623,29 @@ Gate Pipeline verified: 2 submit: true -Zuul Github Support -=================== - -.. code:: yaml - - - pipeline: - name: check - manager: independent - trigger: - github: - - event: pull_request - action: - - opened - - changed - - reopened - success: - github: - status: 'success' - failure: - github: - status: 'failure' - -OpenStack Github Support for Cross Community Testing -==================================================== - - * Github App "OpenStack Zuul" - * App added to github project by project admin - * Project aded to OpenStack's main.yaml - * Test interactions between OpenStack and important adjacent communities - * https://github.com/ansible/ansible/pull/20974 - -Cross Community Testing -======================= - -.. code:: yaml - - - pipeline: - name: check - description: | - Newly uploaded patchsets enter this pipeline to receive an - initial +/-1 Verified vote. - manager: independent - trigger: - gerrit: - - event: patchset-created - - event: change-restored - - event: comment-added - comment: (?i)^(Patch Set [0-9]+:)?( [\w\\+-]*)*(\n\n)?\s*recheck - - event: comment-added - require-approval: - - Verified: [-1, -2] - username: zuul - approval: - - Workflow: 1 - github: - - event: pull_request - action: - - opened - - changed - - reopened - - event: pull_request - action: comment - comment: (?i)^\s*recheck\s*$ - -Cross Community Support cont. -============================= - -.. code:: yaml - - start: - github: - status: pending - comment: false - success: - gerrit: - # Note that gerrit keywords are case-sensitive. - Verified: 1 - github: - status: 'success' - mysql: - failure: - gerrit: - Verified: -1 - github: - status: 'failure' - mysql: - -Cross Source Dependencies -========================= - -.. container:: progressive - - * shade https://review.openstack.org/539563 - - Shift voting flag and test_matrix_branch for ansible-devel job - - Change-ID: Ic9d3983de641dbe618c65b2cbf2dcfa3686575df - - * ansible https://github.com/ansible/ansible/pull/34925 - - continue fact gathering even without dmidecode - - * ansible https://github.com/ansible/ansible/pull/20974 - - Make a generalized OpenStack cloud constructor - - Depends-On: https://review.openstack.org/539563 - Depends-On: https://github.com/ansible/ansible/pull/34925 - Jobs ==== * Jobs run on nodes from nodepool (static or dynamic) * Metadata defined in Zuul's configuration -* Execution content in Ansible (with live streaming!) +* Execution content in Ansible * Jobs may be defined centrally or in the repo being tested * Jobs have contextual variants that simplify configuration -Shared Job Configs -================== - -* Job config repos are all in git -* Designed to support directly sharing job configurations -* git.openstack.org/openstack-infra/zuul-jobs repo is a 'standard library' - to be directly shared between zuul installations - Job === .. code:: yaml - job: - name: base - parent: null - description: | - The base job for Zuul. - timeout: 1800 - nodeset: - nodes: - - name: primary - label: centos-7 - pre-run: playbooks/base/pre.yaml + name: 'base' + timeout: '30m' + nodes: 'ubuntu-xenial' + workspace: '/opt/workspace' + pre-run: + - 'setup-host' post-run: - - playbooks/base/post-ssh.yaml - - playbooks/base/post-logs.yaml - secrets: - - site_logs + - 'archive-logs' Simple Job ========== @@ -919,17 +653,8 @@ Simple Job .. code:: yaml - job: - name: tox - pre-run: playbooks/setup-tox.yaml - run: playbooks/tox.yaml - post-run: playbooks/fetch-tox-output.yaml - - - job: - name: tox-py27 - parent: tox - vars: - tox_envlist: py27 - + name: 'python27' + parent: 'base' Simple Job Variant ================== @@ -937,92 +662,45 @@ Simple Job Variant .. code:: yaml - job: - name: tox-py27 - branches: stable/mitaka - nodeset: - - name: ubuntu-trusty - label: ubuntu-trusty - -Nodesets for Multi-node Jobs -============================ - -.. code:: yaml - - - nodeset: - name: ceph-cluster - nodes: - - name: controller - label: centos-7 - - name: compute1 - label: fedora-26 - - name: compute2 - label: fedora-26 - groups: - - name: ceph-osd - nodes: - - controller - - name: ceph-monitor - nodes: - - controller - - compute1 - - compute2 - + name: 'python27' + branch: 'stable/mitaka' + nodes: 'ubuntu-trusty' + Multi-node Job ============== +.. container:: handout -* nodesets are provided to Ansible for jobs in inventory + nodepool, shrews .. code:: yaml - job: - name: ceph-multinode - nodeset: ceph-cluster - run: playbooks/install-ceph.yaml - -Multi-node Ceph Job Content -=========================== - -.. code:: yaml - - - hosts: all - roles: - - install-ceph - - hosts: ceph-osd - roles: - - start-ceph-osd - - hosts: ceph-monitor - roles: - - start-ceph-monitor - - hosts: all - roles: - - do-something-interesting + name: 'devstack-multinode' + parent: 'base' + nodes: + - name: 'controller' + image: 'ubuntu-xenial' + - name: 'compute' + image: 'ubuntu-xenial' Projects ======== * Projects are git repositories * Specify a set of jobs for each pipeline -* golang git repo naming as been adopted: - -:: - - zuul@ubuntu-xenial:~$ find /home/zuul/src -mindepth 3 -maxdepth 3 -type d - /home/zuul/src/git.openstack.org/openstack-infra/shade - /home/zuul/src/git.openstack.org/openstack/keystoneauth - /home/zuul/src/git.openstack.org/openstack/os-client-config - /home/zuul/src/github.com/ansible/ansible - + Project ======= .. code:: yaml - project: + name: 'nova' check: jobs: - - openstack-tox-py27 - - openstack-tox-py35 - - openstack-doc-build + - python27 + - python35 + - docs Project with Local Variant ========================== @@ -1030,13 +708,14 @@ Project with Local Variant .. code:: yaml - project: + name: 'nova' check: jobs: - - openstack-tox-py27 - - openstack-tox-py35 - - openstack-doc-build - - openstack-tox-pypy: - voting: false + - python27 + - python35 + - docs + - pypy: + voting: false Project with More Local Variants ================================ @@ -1044,15 +723,15 @@ Project with More Local Variants .. code:: yaml - project: - name: openstack/nova + name: 'nova' check: jobs: - - openstack-tox-py27 - - openstack-tox-py35 - - openstack-doc-build: - files: '^docs/.*$' - - openstack-tox-pypy: - voting: false + - python27 + - python35 + - docs: + files: '^docs/.*$' + - pypy: + voting: false Project with Many Local Variants ================================ @@ -1060,106 +739,41 @@ Project with Many Local Variants .. code:: yaml - project: - name: openstack/nova + name: 'nova' check: jobs: - - openstack-tox-py27 - nodeset: - - name: centos-7 - label: centos-7 - - openstack-tox-py27 - branches: stable/newton - nodeset: - - name: ubuntu-trusty - label: ubuntu-trusty - - openstack-doc-build: - files: '^docs/.*$' - - openstack-tox-pypy: - voting: false - -Project With Central and Local Config -===================================== - -.. code:: yaml - - # In git.openstack.org/openstack-infra/project-config: - - project: - name: openstack/nova - templates: - - openstack-tox-jobs - -.. code:: yaml - - # In git.openstack.org/openstack/nova/.zuul.yaml: - - project: - check: - - nova-placement-functional-devstack + - python27: + nodes: 'ubuntu-xenial' + - python27: + branch: 'stable/newton' + nodes: 'ubuntu-trusty' + - python35 + - docs: + files: '^docs/.*$' + - pypy: + voting: false Project with Job Dependencies ============================= .. code:: yaml - # In git.openstack.org/openstack-infra/project-config: - project: - name: openstack/nova + name: nova release: jobs: - - build-artifacts - - upload-tarball: - dependencies: build-artifacts - - upload-pypi: - dependencies: build-artifacts - - notify-mirror: - dependencies: - - upload-tarball - - upload-pypi + - build-tarball: + jobs: + - upload-tarball: + jobs: + - update-mirror Playbooks ========= -* Jobs run Ansible playbooks +* Jobs run playbooks * Playbooks may be defined centrally or in the repo being tested -* Playbooks can use roles from current or other Zuul repos - (or Galaxy, coming soon) -* Playbooks are run on the zuul-executor using bubblewrap - https://github.com/projectatomic/bubblewrap -* Playbooks are not allowed to execute content on 'localhost' - -Job with Roles -============== - -.. code:: yaml - - - job: - name: zuul-integration - description: | - Multi-node Zuul installation and integration test - nodeset: zuul-cluster - roles: - - zuul: openstack-infra/ansible-role-zuul - run: playbooks/zuul-integration - -Job with Multiple Projects -========================== - -.. code:: yaml - - - job: - name: tox-py35-on-zuul - parent: tox-py35 - description: | - Run zuul's py35 unittests on patches to zuul-jobs - vars: - zuul_work_dir: src/git.openstack.org/openstack-infra/zuul - required-projects: - - openstack-infra/zuul - - - project: - name: openstack-infra/zuul-jobs - check: - jobs: - - tox-py35-on-zuul +* Playbooks can use roles from current or other Zuul repos or Galaxy Devstack-gate / Tempest Playbook ================================ @@ -1167,164 +781,42 @@ Devstack-gate / Tempest Playbook .. code:: yaml # devstack-gate / tempest playbook + --- hosts: all roles: - - setup-multinode-networking - - partition-swap - - configure-mirrors - - run-devstack - - run-tempest + - setup-multinode-networking + - partition-swap + - configure-mirrors + - run-devstack + - run-tempest Simple Shell Playbook ===================== .. code:: yaml + --- hosts: controller - tasks: - - shell: ./run_tests.sh + roles: + - shell: | + cd $WORKSPACE + ./run_tests.sh + Test Like Production ==================== If you use Ansible for deployment, your test and deployment processes and playbooks are the same - -What if you don't use Ansible? -============================== - -OpenStack Infra Control Plane uses Puppet -========================================= - -.. code:: yaml - - # In git.openstack.org/openstack-infra/project-config/roles/legacy-install-afs-with-puppet/tasks/main.yaml - - name: Install puppet - shell: ./install_puppet.sh - args: - chdir: "{{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/system-config" - environment: - # Skip setting up pip, our images have already done this. - SETUP_PIP: "false" - become: yes - - - name: Copy manifest - copy: - src: manifest.pp - dest: "{{ ansible_user_dir }}/manifest.pp" - - - name: Run puppet - puppet: - manifest: "{{ ansible_user_dir }}/manifest.pp" - become: yes - -Secrets -======= - -* Inspired by Kubernetes Secrets API -* Projects can add named encrypted secrets to their .zuul.yaml file -* Jobs can request to use secrets by name -* Jobs using secrets are not reconfigured speculatively -* Secrets can only be used by the same project they are defined in -* Public key per project: - ``{{ zuul_url }}/{{ tenant }}/{{ project }}.pub`` - -:: - GET http://zuul.openstack.org/openstack-infra/shade.pub - -Secret Example (note, no admins had to enable this) -=================================================== - -.. code:: yaml - - # In git.openstack.org/openstack/loci/.zuul.yaml: - - secret: - name: loci_docker_login - data: - user: !encrypted/pkcs1-oaep - - r8Nbpq5olmfLF035BZ/CUoFLIdhvBi/49KuochOAHbvns+xMiho3C7MEFzYDqJX3IhHde - BICYOgK7qnyINOIZL2e7pl75rEdHQwJjSFUMkpdY6wEP7f9hpolj9xVp0ifHUVQqPHMRn - zoPFd8MEAHxH5GLmc2SWJ98E/QUqGltxBi1YRSZoCcNtq3tHFK5Y+xQlLhIseJ2HkpDs6 - YXOGP9Qt4Va6sdyBcA90H+apSAcYA3Duu962ySZQAsYNui/3NQq3gLA+OZeyTJtcrh4hj - Rb5dBnDWfSrMpxdNkbPXXgbQaxO3T0L4jbaOF8VKEsiI9olBrOeV2M9ddYJjSsHGj4XR8 - 4vwS0+doB7np93fujiDuHVgdG8R40NW2GznyKRlRtzAORla7Mzw1Y1MokcUyY6p1LlLLl - wUuWYCCEuRciOPhZXQ2u42qju/zrK2/dPnO8HfUINSrN0WbNq14ZwPpbj0ro02oGPbtwu - OTw1z+N0Nc+GuLWlwYJGYM/z0UnvDR3WEBc2kXbVev9w4n0cB3RyphML2PDZZWbw8tjnX - h1VsAOJ0Qo4qq1K/ft95ypd+vtjkfepEgHEBmJNwutJa9IHAkGfrkO9VkpUTPpfffnPwz - d0/zaaadNl6MLQUSutRwY23YIIbv+fmukxw2vnJmvn6abkBlMya7KgtifwNA8c= - password: !encrypted/pkcs1-oaep - - gUEX4eY3JAk/Xt7Evmf/hF7xr6HpNRXTibZjrKTbmI4QYHlzEBrBbHey27Pt/eYvKKeKw - hk8MDQ4rNX7ZK1v+CKTilUfOf4AkKYbe6JFDd4z+zIZ2PAA7ZedO5FY/OnqrG7nhLvQHE - 5nQrYwmxRp4O8eU5qG1dSrM9X+bzri8UnsI7URjqmEsIvlUqtybQKB9qQXT4d6mOeaKGE - 5h6Ydkb9Zdi4Qh+GpCGDYwHZKu1mBgVK5M1G6NFMy1DYz+4NJNkTRe9J+0TmWhQ/KZSqo - 4ck0x7Tb0Nr7hQzV8SxlwkaCTLDzvbiqmsJPLmzXY2jry6QsaRCpthS01vnj47itoZ/7p - taH9CoJ0Gl7AkaxsrDSVjWSjatTQpsy1ub2fuzWHH4ASJFCiu83Lb2xwYts++r8ZSn+mA - hbEs0GzPI6dIWg0u7aUsRWMOB4A+6t2IOJibVYwmwkG8TjHRXxVCLH5sY+i3MR+NicR9T - IZFdY/AyH6vt5uHLQDU35+5n91pUG3F2lyiY5aeMOvBL05p27GTMuixR5ZoHcvSoHHtCq - 7Wnk21iHqmv/UnEzqUfXZOque9YP386RBWkshrHd0x3OHUfBK/WrpivxvIGBzGwMr2qAj - /AhJsfDXKBBbhGOGk1u5oBLjeC4SRnAcIVh1+RWzR4/cAhOuy2EcbzxaGb6VTM= - -Secret Example -============== - -.. code:: yaml - - # In git.openstack.org/openstack/loci/.zuul.yaml: - - job: - name: publish-loci-cinder - parent: loci-cinder - post-run: playbooks/push - secrets: - - loci_docker_login - - # In git.openstack.org/openstack/loci/playbooks/push.yaml: - - hosts: all - tasks: - - include_vars: vars.yaml - - - name: Push project to DockerHub - block: - - command: docker login -u {{ loci_docker_login.user }} -p {{ loci_docker_login.password }} - no_log: True - - command: docker push openstackloci/{{ project }}:{{ branch }}-{{ item.name }} - with_items: "{{ distros }}" - -Status -====== - -* Zuul v3 is in production for OpenStack (in OpenStack VMs) -* Zuul v3 also runing at BMW (in OpenShift) and Godaddy (in kuberenetes) - and Huawei's OpenLab. -* Software Factory updated to v3 - https://softwarefactory-project.io/sf/welcome.html -* will tag and release v3.0 once we're satisfied it's good for other people - (within the next few weeks) - -What's Next? -============ - -* shared job doc generation -* node providers - * kuberenetes - * OCI/docker - * Mac Stadium (for our Ansible friends) - * ec2 - * ansible -* support for galaxy roles -* in-line code-review comments from Zuul -* native container/kubernetes job execution - + Important Links =============== * https://zuul-ci.org/ -* https://git.openstack.org/cgit/openstack-infra/zuul -* https://docs.openstack.org/infra/zuul -* https://docs.openstack.org/infra/manual/zuulv3.html -* https://docs.openstack.org/infra/zuul-jobs/ +* https://git.zuul-ci.org/cgit/zuul +* https://zuul-ci.org/docs/zuul +* https://zuul-ci.org/docs/zuul-jobs/ * https://docs.openstack.org/infra/openstack-zuul-jobs/ -* https://storyboard.openstack.org/#!/project/679 -* https://storyboard.openstack.org/#!/board/41 * freenode:#zuul Questions -- cgit v1.2.3