diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/zuulv3/images/architecture.ans | 38 | ||||
-rw-r--r-- | src/zuulv3/images/orig/title.ans | 40 | ||||
-rw-r--r-- | src/zuulv3/images/title.ans | 44 | ||||
-rw-r--r-- | src/zuulv3/images/zuul.ans | 31 | ||||
-rwxr-xr-x | src/zuulv3/start.sh | 6 | ||||
-rw-r--r-- | src/zuulv3/zuul.rst | 948 |
6 files changed, 298 insertions, 809 deletions
diff --git a/src/zuulv3/images/architecture.ans b/src/zuulv3/images/architecture.ans index 0698ccc..ad308eb 100644 --- a/src/zuulv3/images/architecture.ans +++ b/src/zuulv3/images/architecture.ans | |||
@@ -4,42 +4,42 @@ | |||
4 | [A[79C | 4 | [A[79C |
5 | 5 | ||
6 | [A[79C | 6 | [A[79C |
7 | [1m┌─────────────┐[0m | 7 | [1m┌─────────────┐[0m |
8 | [A[79C | 8 | [A[79C |
9 | [1m┌───────┤[0m [1;31mZuul Merger[0;37m [1m│[0m | 9 | [1m┌───────┤[0m [1;31mZuul Merger[0;37m [1m│[0m [1;31m [37m [0m |
10 | [A[79C | 10 | [A[79C |
11 | [1m│[0m [1m└──────┬──────┘[0m | 11 | [1m┌────┴───┐[0m [1m└──┬───┬──────┘[0m |
12 | [A[79C | 12 | [A[79C |
13 | [1m│[0m [1m│[0m | 13 | [1m┌──┤[0m [1;33mGerrit[0;37m [1m├──┐[0m [1m│[0m [1m│[0m |
14 | [A[79C | 14 | [A[79C |
15 | [1m│[0m [1m│[0m | 15 | [1m│[0m [1m└────────┘[0m [1m│[0m [1m┌─┘[0m [1m│[0m |
16 | [A[79C | 16 | [A[79C |
17 | [1m ┌────┴───┐[0m [1m┌──┴───┐[0m [1m┌───────────────┐[0m | 17 | [1m [0m [1m│ ┌────────┐[0m [1m│[0m [1m│[0m [1m┌──┴───┐[0m [1m┌───────────────┐[0m |
18 | [A[79C | 18 | [A[79C |
19 | [1m┌───────┤[0m [1;33mGerrit[0;37m [1m├───────┤[0m [1;31mZuul[0;37m [1m├─────┤[0m [1;31mZuul Executor[0;37m [1m│[0m | 19 | [1m┌─────┼──┤[0m [1;31m Web [0;37m [1m├──┼────┤[0m [1;31mZuul[0;37m [1m├─────┤[0m [1;31mZuul Executor[0;37m [1m│ [0m |
20 | [A[79C | 20 | [A[79C |
21 | ____ [1m└────────┘[0m [1m└──┬───┘[0m [1m│[0m [1m┌───────┐[0m [1m│[0m | 21 | ____ [1m│[0m [1m└────┬───┘[0m [1m│[0m [1m│[0m [1m└──┬───┘[0m [1m│[0m [1m┌───────┐[0m [1m│[0m |
22 | [A[79C | 22 | [A[79C |
23 | |[32m... [37m| [1m│[0m [1m│ [0m [1m│[31mAnsible[37m│ │ [0m | 23 | |[32m... [37m| [1m│[0m [1m┌────┴───┐[0m [1m│ │[0m [1m│[0m [1m│ [0m [1m│[31mAnsible[37m│ │[0m |
24 | [A[79C | 24 | [A[79C |
25 | |[32m... [37m| [1m│[0m [1m└───┴───┬───┴───┘[0m | 25 | |[32m... [37m| [1m├──┤[0m [1;33mGitHub[0;37m [1m├──┼─┘[0m [1m│[0m [1m└───┴───┬───┴───┘[0m |
26 | [A[79C | 26 | [A[79C |
27 | /______\ [1m┌────┴─────┐[0m [1m│[0m | 27 | /______\ [1m│[0m [1m└────────┘[0m [1m│[0m [1m┌────┴─────┐[0m [1m│[0m |
28 | [A[79C | 28 | [A[79C |
29 | [1mo[0m [1m│[0m [1;34mNodepool[0;37m [1m│[0m [1m│[0m | 29 | [1mo[0m [1m│[0m [1m┌────────┐[0m [1m│[0m [1m│[0m [1;34mNodepool[0;37m [1m│[0m [1m│[0m |
30 | [A[79C | 30 | [A[79C |
31 | [1m-|-[0m [1m [0m [1m└────┬─────┘[0m [1m│[0m | 31 | [1m-|-[0m [1m└──┤[31m Finger [37m├──┘[0m [1m└────┬─────┘[0m [1m│[0m |
32 | [A[79C | 32 | [A[79C |
33 | [1m/ \[0m [1m│[0m [1m│[0m | 33 | [1m/ \[0m [1m└────────┘[0m [1m│[0m [1m│[0m |
34 | [A[79C | 34 | [A[79C |
35 | [1mDev[0m [1;44mCloud │ [0;40m [1m│[0m | 35 | [1mDev[0m [1;44mCloud │ [0;40m [1m│[0m |
36 | [A[79C | 36 | [A[79C |
37 | [1;44m Node 1 ───[40m──────────┤[0m | 37 | [1;44m Node 1 ───[40m──────────┤[0m |
38 | [A[79C | 38 | [A[79C |
39 | [1;44m ... [0;40m [1m│[0m | 39 | [1;44m ... [0;40m [1m│[0m |
40 | [A[79C | 40 | [A[79C |
41 | [1;44m Node 2 ───[40m──────────┘[0m | 41 | [1;44m Node 2 ───[40m──────────┘[0m |
42 | [A[79C | 42 | [A[79C |
43 | [1;30m(Not to scale)[0;37m [1;44m [0;40m | 43 | [1;30m(Not to scale)[0;37m [1;44m [0;40m |
44 | [A[79C | 44 | [A[79C |
45 | [0m \ No newline at end of file | 45 | [0m \ No newline at end of file |
diff --git a/src/zuulv3/images/orig/title.ans b/src/zuulv3/images/orig/title.ans index 82ef305..8442c58 100644 --- a/src/zuulv3/images/orig/title.ans +++ b/src/zuulv3/images/orig/title.ans | |||
@@ -1,42 +1,40 @@ | |||
1 | [0m [1;34m______________[0;37m [1;34m..[0;37m [1;34m..[0;37m [1;34m______________[0;37m | 1 | [0m[1;33m͵[0;37m |
2 | [A[79C | 2 | [A[79C |
3 | [1;34m( ( [0;37m [1;34m\ \_____)____(_____/ /[0;37m [1;34m ) )[0;37m | 3 | [1;33mĴ[0;37m |
4 | [A[79C | 4 | [A[79C |
5 | [1;34m\ )[0;37m [1;34m`' [0;37m [1;34m `'[0;37m [1;34m( /[0;37m | 5 | [33m[1m[0;33m[37m |
6 | [A[79C | 6 | [A[79C |
7 | [1;34m)/ [0;37m [1;34m \([0;37m | 7 | |
8 | [A[79C | ||
9 | [1;34m /' [0;37m [1mREST APIs and the Return of the[0m [1;34m `\ [0;37m | ||
10 | [A[79C | 8 | [A[79C |
11 | [1;34mO [0;37m [1;32m ___ ___ _ _ ___ ___ _ ___ [0;37m [1;34m O[0;37m | 9 | |
12 | [A[79C | 10 | [A[79C |
13 | [1;34m[0;37m [1;32m / __/ _ \| \| / __|/ _ \| | | __|[0;37m [1;34m[0;37m | 11 | |
14 | [A[79C | 12 | [A[79C |
15 | [1;34m[0;37m [1;32m| (_| (_) | .` \__ \ (_) | |__| _| [0;37m [1;34m[0;37m | 13 | [32m _____ _[37m |
16 | [A[79C | 14 | [A[79C |
17 | [1;34m[0;37m [1;32m \___\___/|_|\_|___/\___/|____|___|[0;37m [1;34m[0;37m | 15 | [32m|__ / _ _ _| |[37m |
18 | [A[79C | 16 | [A[79C |
19 | [1;34m[0;37m [1;34m[0;37m | 17 | [32m/ / | | | | | | |[37m |
20 | [A[79C | 18 | [A[79C |
21 | [1;34m[0;37m [1mApplication[0m [1;34m[0;37m | 19 | [32m/ /| |_| | |_| | |[37m |
22 | [A[79C | 20 | [A[79C |
23 | [1;34m[0;37m [1;34m[0;37m | 21 | [32m/____\__,_|\__,_|_|[37m |
24 | [A[79C | 22 | [A[79C |
25 | [1;34mO [0;37m [33mJames E. Blair <corvus@inaugust.com>[37m [1;34m O[0;37m | 23 | |
26 | [A[79C | 24 | [A[79C |
27 | [1;34m \.[0;37m [1;34m./ [0;37m | 25 | |
28 | [A[79C | 26 | [A[79C |
29 | [1;34m)\ [0;37m [1;34m,____[0;37m [1;34m____,[0;37m [1;34m /([0;37m | 27 | |
30 | [A[79C | 28 | [A[79C |
31 | [1;34m/ )[0;37m [1;34m/ ____\ ____ /____ \[0;37m [1;34m( \[0;37m | 29 | |
32 | [A[79C | 30 | [A[79C |
33 | [1;34m( (_________/_/ ) ( \_\_________) )[0;37m | 31 | |
34 | [A[79C | 32 | [A[79C |
35 | [1;34m ~-' [0;37m [1;34m`'[0;37m [1;34m`'[0;37m [1;34m `-~ [0;37m | 33 | [33m[1m[0;33m[37m |
36 | [A[79C | 34 | [A[79C |
37 | 35 | [1;33mĴ[0;37m | |
38 | [A[79C | 36 | [A[79C |
39 | 37 | [1;33m͵[0;37m | |
40 | [A[79C | 38 | [A[79C |
41 | 39 | ||
42 | [A[79C | 40 | [A[79C |
diff --git a/src/zuulv3/images/title.ans b/src/zuulv3/images/title.ans index c784327..8857086 100644 --- a/src/zuulv3/images/title.ans +++ b/src/zuulv3/images/title.ans | |||
@@ -1,45 +1,47 @@ | |||
1 | [0m [1;34m______________[0;37m [1;34m.──.[0;37m [1;34m.──.[0;37m [1;34m______________[0;37m | 1 | [0m[1;33m══════════════════════╡│││╞═════════════════════[0;37m |
2 | [A[79C | 2 | [A[79C |
3 | [1;34m( ( [0;37m [1;34m\ \_____)____(_____/ /[0;37m [1;34m ) )[0;37m | 3 | [1;33m────────────────────┤│├───────────────────[0;37m |
4 | [A[79C | 4 | [A[79C |
5 | [1;34m\ )[0;37m [1;34m`─────' [0;37m [1;34m `─────'[0;37m [1;34m( /[0;37m | 5 | [33m──────────────────[1m│[0;33m─────────────────[37m |
6 | [A[79C | 6 | [A[79C |
7 | [1;34m)/ [0;37m [1;34m \([0;37m | 7 | |
8 | [A[79C | ||
9 | [1;34m /' [0;37m [1mREST APIs and the Return of the[0m [1;34m `\ [0;37m | ||
10 | [A[79C | 8 | [A[79C |
11 | [1;34mO [0;37m [1;32m ___ ___ _ _ ___ ___ _ ___ [0;37m [1;34m O[0;37m | 9 | |
12 | [A[79C | 10 | [A[79C |
13 | [1;34m│[0;37m [1;32m / __/ _ \| \| / __|/ _ \| | | __|[0;37m [1;34m│[0;37m | 11 | |
14 | [A[79C | 12 | [A[79C |
15 | [1;34m│[0;37m [1;32m| (_| (_) | .` \__ \ (_) | |__| _| [0;37m [1;34m│[0;37m | 13 | [32m _____ _[37m |
16 | [A[79C | 14 | [A[79C |
17 | [1;34m│[0;37m [1;32m \___\___/|_|\_|___/\___/|____|___|[0;37m [1;34m│[0;37m | 15 | [32m|__ / _ _ _| |[37m |
18 | [A[79C | 16 | [A[79C |
19 | [1;34m│[0;37m [1;34m│[0;37m | 17 | [32m/ / | | | | | | |[37m |
20 | [A[79C | 18 | [A[79C |
21 | [1;34m│[0;37m [1mApplication[0m [1;34m│[0;37m | 19 | [32m/ /| |_| | |_| | |[37m |
22 | [A[79C | 20 | [A[79C |
23 | [1;34m│[0;37m [1;34m│[0;37m | 21 | [32m/____\__,_|\__,_|_|[37m |
24 | [A[79C | 22 | [A[79C |
25 | [1;34mO [0;37m [33mJames E. Blair <corvus@inaugust.com>[37m [1;34m O[0;37m | 23 | |
26 | [A[79C | 24 | [A[79C |
27 | [1;34m \.[0;37m [1;34m./ [0;37m | 25 | |
28 | [A[79C | 26 | [A[79C |
29 | [1;34m)\ [0;37m [1;34m,____[0;37m [1;34m____,[0;37m [1;34m /([0;37m | 27 | Monty Taylor |
30 | [A[79C | 28 | [A[79C |
31 | [1;34m/ )[0;37m [1;34m/ ____\ ____ /____ \[0;37m [1;34m( \[0;37m | 29 | irc: mordred |
32 | [A[79C | 30 | [A[79C |
33 | [1;34m( (_________/_/ ) ( \_\_________) )[0;37m | 31 | twitter: @e_monty |
34 | [A[79C | 32 | [A[79C |
35 | [1;34m ~-' [0;37m [1;34m`──'[0;37m [1;34m`──'[0;37m [1;34m `-~ [0;37m | 33 | Red Hat |
36 | [A[79C | 34 | [A[79C |
37 | 35 | ||
38 | [A[79C | 36 | [A[79C |
39 | 37 | [33m──────────────────[1m│[0;33m─────────────────[37m | |
38 | [A[79C | ||
39 | [1;33m────────────────────┤│├───────────────────[0;37m | ||
40 | [A[79C | ||
41 | [1;33m══════════════════════╡│││╞═════════════════════[0;37m | ||
40 | [A[79C | 42 | [A[79C |
41 | 43 | ||
42 | [A[79C | 44 | [A[79C |
43 | 45 | ||
44 | [A[79C | 46 | [A[79C |
45 | [0m \ No newline at end of file | 47 | [0m |
diff --git a/src/zuulv3/images/zuul.ans b/src/zuulv3/images/zuul.ans index 6369083..44094b8 100644 --- a/src/zuulv3/images/zuul.ans +++ b/src/zuulv3/images/zuul.ans | |||
@@ -9,28 +9,25 @@ | |||
9 | 9 | ||
10 | [A[79C | 10 | [A[79C |
11 | 11 | ||
12 | [A[79C | 12 | [34;1m |
13 | [32m _____ _[37m | 13 | ╱╲ |
14 | [A[79C | 14 | ╱ ╲ |
15 | [32m|__ / _ _ _| |[37m | 15 | ──────────────── |
16 | [A[79C | 16 | ╲┌─┬────────┬─┐╱ |
17 | [32m/ / | | | | | | |[37m | 17 | ├─┼────────┼─┤ |
18 | [A[79C | 18 | ╱│ │ ▏ │ │╲ |
19 | [32m/ /| |_| | |_| | |[37m | 19 | ╱ │ │ ▏ │ │ ╲ |
20 | [A[79C | 20 | ╱ │ │ ▏ │ │ ╲ |
21 | [32m/____\__,_|\__,_|_|[37m | 21 | ╱ │ │ ▏ │ │ ╲ |
22 | [A[79C | 22 | ────┴─┴────────┴─┴──── |
23 | 23 | ||
24 | [A[79C | 24 | Z U U L |
25 | |||
26 | [A[79C | 25 | [A[79C |
27 | 26 | ||
28 | [A[79C | 27 | [A[79C |
29 | 28 | ||
30 | [A[79C | 29 | [A[79C |
31 | 30 | [0;33m──────────────────[1m│[0;33m─────────────────[37m | |
32 | [A[79C | ||
33 | [33m──────────────────[1m│[0;33m─────────────────[37m | ||
34 | [A[79C | 31 | [A[79C |
35 | [1;33m────────────────────┤│├───────────────────[0;37m | 32 | [1;33m────────────────────┤│├───────────────────[0;37m |
36 | [A[79C | 33 | [A[79C |
diff --git a/src/zuulv3/start.sh b/src/zuulv3/start.sh index 9eaf8b6..31b2cee 100755 --- a/src/zuulv3/start.sh +++ b/src/zuulv3/start.sh | |||
@@ -1,8 +1,8 @@ | |||
1 | #!/bin/sh | 1 | #!/bin/sh |
2 | 2 | ||
3 | mate-terminal --geometry 68x24 -x presentty $(pwd)/zuul.rst & | 3 | gnome-terminal --geometry 68x24 -- ~/presentty/venv/bin/presentty zuul.rst & |
4 | #geeqie -t images & | 4 | geeqie -t images & |
5 | 5 | ||
6 | sleep 1 | 6 | sleep 1 |
7 | 7 | ||
8 | mate-terminal --maximize -x presentty-console $(pwd)/zuul.rst & | 8 | gnome-terminal --maximize -- ~/presentty/venv/bin/presentty-console zuul.rst & |
diff --git a/src/zuulv3/zuul.rst b/src/zuulv3/zuul.rst index 3ce2437..b38675b 100644 --- a/src/zuulv3/zuul.rst +++ b/src/zuulv3/zuul.rst | |||
@@ -4,7 +4,7 @@ | |||
4 | .. pygments yaml? (only file breaks (---) tinted) | 4 | .. pygments yaml? (only file breaks (---) tinted) |
5 | .. slide on high level v3 changes | 5 | .. slide on high level v3 changes |
6 | .. slide on nodepool | 6 | .. slide on nodepool |
7 | 7 | ||
8 | .. transition:: dissolve | 8 | .. transition:: dissolve |
9 | :duration: 0.4 | 9 | :duration: 0.4 |
10 | 10 | ||
@@ -23,43 +23,23 @@ Preshow | |||
23 | Zuul | 23 | Zuul |
24 | ==== | 24 | ==== |
25 | .. hidetitle:: | 25 | .. hidetitle:: |
26 | .. ansi:: images/zuul.ans | 26 | .. ansi:: images/title.ans |
27 | |||
28 | Meta Information | ||
29 | ================ | ||
30 | |||
31 | * This Talk is Free Software: http://git.inaugust.com/cgit/inaugust.com/tree/src/zuulv3 | ||
32 | * Twitter: @e_monty | ||
33 | * Freenode: mordred | ||
34 | * email: mordred@inaugust.com | ||
35 | 27 | ||
36 | Red Hat | 28 | Red Hat |
37 | ======= | 29 | ======= |
38 | |||
39 | .. hidetitle:: | 30 | .. hidetitle:: |
40 | .. container:: handout | 31 | .. container:: handout |
41 | 32 | i work for | |
42 | * I work for Red Hat in the Office of Technology as the Chief Architect | ||
43 | for CI/CD | ||
44 | 33 | ||
45 | .. ansi:: images/redhat.ans | 34 | .. ansi:: images/redhat.ans |
46 | 35 | ||
47 | OpenStack | 36 | OpenStack |
48 | ========= | 37 | ========= |
49 | .. container:: handout | ||
50 | |||
51 | * I work on OpenStack. | ||
52 | * I sit on the Technical Committee. I was on the Board of Directors | ||
53 | |||
54 | .. hidetitle:: | 38 | .. hidetitle:: |
55 | .. ansi:: images/openstack.ans | 39 | .. ansi:: images/openstack.ans |
56 | 40 | ||
57 | OpenStack Infra | 41 | OpenStack Infra |
58 | =============== | 42 | =============== |
59 | .. container:: handout | ||
60 | |||
61 | * My primary technical role with OpenStack is working on the OpenStack CI | ||
62 | system. | ||
63 | 43 | ||
64 | :: | 44 | :: |
65 | 45 | ||
@@ -73,98 +53,125 @@ OpenStack Infra | |||
73 | 53 | ||
74 | Zuul | 54 | Zuul |
75 | ==== | 55 | ==== |
76 | .. container:: handout | ||
77 | |||
78 | * As part of working on OpenStack Infra I work on Zuul | ||
79 | |||
80 | .. hidetitle:: | 56 | .. hidetitle:: |
81 | .. ansi:: images/zuul.ans | 57 | .. ansi:: images/zuul.ans |
82 | 58 | ||
83 | 59 | ||
84 | Ansible | 60 | Ansible |
85 | ======= | 61 | ======= |
86 | |||
87 | .. container:: handout | ||
88 | |||
89 | * And as part of working on Zuul and on OpenStack I work on Ansible | ||
90 | * I maintain the OpenStack modules for Ansible as well as the shade library | ||
91 | |||
92 | .. hidetitle:: | 62 | .. hidetitle:: |
93 | .. ansi:: images/ansible.ans | 63 | .. ansi:: images/ansible.ans |
94 | 64 | ||
95 | Presentation Checklist | 65 | Presentation Checklist |
96 | ====================== | 66 | ====================== |
97 | 67 | ||
98 | .. container:: handout | ||
99 | |||
100 | * Every good presentation needs logos, so we're starting well | ||
101 | |||
102 | :: | 68 | :: |
103 | 69 | ||
104 | [X] Logos | 70 | [X] Logos |
105 | 71 | ||
106 | |||
107 | Spoilers | 72 | Spoilers |
108 | ======== | 73 | ======== |
109 | 74 | ||
110 | * What the old version of Zuul (v2) was | 75 | * What Zuul v3 does |
111 | 76 | ||
112 | * a nifty project gating system | 77 | * multiple repositories |
78 | * integrated deliverable | ||
79 | * gated commits | ||
80 | * open tooling | ||
81 | * nobody is special | ||
82 | * testing like deployment | ||
113 | 83 | ||
114 | * What the new version of Zuul (v3) is | 84 | OpenStack Is |
85 | ============ | ||
115 | 86 | ||
116 | * multinode support | 87 | * Federated |
117 | * live configuration changes | 88 | * Distributed |
118 | * better job definition | 89 | * Large |
119 | * sharable job definition | 90 | * Open |
120 | * testing like deployment | 91 | * Not Alone |
121 | 92 | ||
122 | What do I mean by Massive Scale? | 93 | Federated |
123 | ================================ | 94 | ========= |
124 | 95 | ||
125 | * Contributors (~2k / 6 month period) | 96 | * Hundreds of involved companies |
126 | * Companies | 97 | * No 'main' company |
127 | * Changes | 98 | * "Decisions are made by those who show up" |
128 | * Code Repositories (1868 as of this morning) | 99 | * Union of priorities/use cases |
129 | * Communities | ||
130 | 100 | ||
131 | OpenStack Scale Comparison | 101 | Impact of being Federated |
102 | ========================= | ||
103 | |||
104 | * No company can appoint people to positions in the project | ||
105 | * The project cannot fire anyone | ||
106 | * Variable background of contributors | ||
107 | * Heavy reliance on consensus | ||
108 | |||
109 | Distributed | ||
110 | =========== | ||
111 | |||
112 | * There is no office | ||
113 | * Contributor base is global | ||
114 | * Multitude of contributor backgrounds | ||
115 | |||
116 | Impact of being Distributed | ||
132 | =========================== | 117 | =========================== |
133 | 118 | ||
134 | * 2KJPH (2,000 jobs per hour) | 119 | * Tooling must empower all contributors, regardless of background, |
135 | * Nodes from 12 Regions of 5 Public and 1 Private OpenStack Clouds | 120 | skill level or cultural context |
136 | (Thanks Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro) | 121 | * Heavy preference for text-based communication |
137 | * 10,000 changes merged per month | 122 | * Cannot assume US-centric needs or solutions |
123 | |||
124 | Large numbers of | ||
125 | ================ | ||
126 | |||
127 | * Contributors (\~2k in any given 6 month period) | ||
128 | * Changes | ||
129 | * Code Repositories (1955 as of this morning) | ||
138 | 130 | ||
139 | OpenStack Scale Comparison | 131 | OpenStack Scale Comparison |
140 | ========================== | 132 | ========================== |
141 | 133 | ||
142 | * 2KJPH (2,000 jobs per hour) | 134 | * 2KJPH (2,000 jobs per hour) |
143 | * Nodes from 12 Regions of 5 Public and 1 Private OpenStack Clouds | 135 | * Build Nodes from 13 Regions of 5 Public and 2 Private OpenStack Clouds |
144 | (Thanks Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro) | 136 | * Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro, Limestone |
145 | * 10,000 changes merged per month | 137 | * 10,000 changes merged per month |
146 | 138 | ||
147 | * By comparison, our friends at the amazing project Ansible received | 139 | Four Opens |
148 | 13,000 changes and had merged 8,000 of them in its first 4 years. | 140 | ========== |
149 | 141 | ||
150 | Pretty Things to Look for Scale | 142 | * Open Source |
151 | =============================== | 143 | (we don't hold back Enterprise features, we don't cripple things) |
144 | * Open Design | ||
145 | (design process open to all, decisions are not made inside company doors) | ||
146 | * Open Development | ||
147 | (public source code, public code review, all code is reviewed and gated) | ||
148 | * Open Community | ||
149 | (lazy consensus, democratic leadership from participants, | ||
150 | public logged meetings in IRC, public archived mailing lists) | ||
151 | |||
152 | We're Not Alone | ||
153 | =============== | ||
152 | 154 | ||
153 | * http://grafana.openstack.org/dashboard/db/zuul-status | 155 | * Dependencies (libvirt/kvm/xen, mysql/pg, rabbit, |
154 | * http://grafana.openstack.org/dashboard/db/nodepool | 156 | python/javascript, ceph/gluster, ansible/salt/puppet/chef, ovs/odl) |
155 | * http://zuul.openstack.org/ | 157 | * Adjacencies (kubernetes, ansible, terraform, opnfv, spinnaker) |
158 | * Vendors (plugins, products, services, distros) | ||
156 | 159 | ||
157 | Dealing With Scale | 160 | Developer Process In a Nutshell |
158 | ================== | 161 | =============================== |
159 | 162 | ||
160 | * Egalitarian Process | 163 | * Code Review - nobody has direct commit/push access |
161 | * Balance Centralized vs Distributed | 164 | * 3rd-Party CI for vendors |
162 | * Code Review plus Enforced Testing | 165 | * Gated Commits |
163 | 166 | ||
164 | OpenStack Developer Workflow | 167 | OpenStack Developer Workflow |
165 | ============================ | 168 | ============================ |
166 | .. container:: handout | 169 | .. container:: handout |
167 | 170 | ||
171 | * Who has submitted a patch? | ||
172 | * Who wants to? | ||
173 | * (Who is here because the name of this talk is weird?) | ||
174 | |||
168 | :: | 175 | :: |
169 | 176 | ||
170 | Hack Review Test | 177 | Hack Review Test |
@@ -187,39 +194,18 @@ Gerrit | |||
187 | .. hidetitle:: | 194 | .. hidetitle:: |
188 | .. container:: handout | 195 | .. container:: handout |
189 | 196 | ||
190 | the primary interface for our developers is the code review system | ||
191 | gerrit. No matter how complex zuul becomes, this is still primary | ||
192 | focus we want the developers to have. | ||
193 | |||
194 | explain patch upload, zuul runs, test results displayed in gerrit | 197 | explain patch upload, zuul runs, test results displayed in gerrit |
195 | this is all the interface to zuul users need to see | 198 | this is all the interface to zuul users need to see |
196 | 199 | ||
200 | switch to actual gertty screenshot | ||
201 | |||
202 | also show zuul status page | ||
203 | |||
197 | but zuul is doing a lot of work behind the scenes, and if you look | 204 | but zuul is doing a lot of work behind the scenes, and if you look |
198 | closer, this is what you see | 205 | closer, this is what you see |
199 | 206 | ||
200 | .. ansi:: images/color-gertty.ans | 207 | .. ansi:: images/color-gertty.ans |
201 | 208 | ||
202 | Github Developer Workflow | ||
203 | ========================= | ||
204 | .. container:: handout | ||
205 | |||
206 | :: | ||
207 | |||
208 | Hack Review Test | ||
209 | ========= ========== ========== | ||
210 | |||
211 | push approve | ||
212 | +-------------+ +-------------+ | ||
213 | | | | | | ||
214 | +------+--+ +--v----+--+ +--v-------+ | ||
215 | | | | | | | | ||
216 | | $EDITOR | | Github | | Zuul | | ||
217 | | | | | | | | ||
218 | +------^--+ +--+----^--+ +--+-------+ | ||
219 | | | | | | ||
220 | +-------------+ +-------------+ | ||
221 | clone merge | ||
222 | |||
223 | Zuul Architecture | 209 | Zuul Architecture |
224 | ================= | 210 | ================= |
225 | 211 | ||
@@ -236,28 +222,37 @@ Presentation Checklist | |||
236 | Nodepool | 222 | Nodepool |
237 | ======== | 223 | ======== |
238 | 224 | ||
239 | .. container:: handout | 225 | * A separate program that works very closely with *Zuul* |
226 | * Builds images daily and uploads to clouds | ||
227 | * Creates and destroys (at least) a vm for every job | ||
240 | 228 | ||
241 | nodepool builds nodes for zuul | 229 | (Remember that 2,000 jobs per hour number?) |
242 | Remember that 2,000 jobs per hour number? | 230 | |
243 | Each job gets a fresh VM - that's 2,000 VMs per hours | 231 | Not just for OpenStack |
244 | Treats our 12 regions across 6 clouds as one REALLY big cloud | 232 | ====================== |
245 | 233 | ||
246 | :: | 234 | * Zuul v3 is in production for OpenStack (in OpenStack VMs) |
247 | 235 | ||
248 | * A separate program that works very closely with *zuul* | 236 | Also running at: |
249 | * Builds images daily and uploads to clouds | ||
250 | * Creates and destroys (at least) a VM for every job | ||
251 | * Supports using pre-existing nodes (static provider) | ||
252 | 237 | ||
253 | Nodepool can use pre-existing images, BUT ... | 238 | * BMW (control plane in OpenShift) |
254 | ============================================= | 239 | * Easystack |
240 | * GoDaddy (control plane in Kubernetes) | ||
241 | * OpenContrail | ||
242 | * OpenLab | ||
243 | * Red Hat | ||
244 | * others ... | ||
255 | 245 | ||
256 | * Clouds have 'helpful' differences between base images | 246 | Zuul in a nutshell |
257 | * Cloud images have 'helpful' software pre-installed | 247 | ================== |
258 | * Distros have 'helpful' different user names | 248 | |
259 | * Most clouds use DHCP for networking, but some don't | 249 | * Listens for code events |
260 | * We can add pre-cached content | 250 | * Prepares appropriate job config and git repo states |
251 | * Allocates nodes for test jobs | ||
252 | * Pushes git repo states to nodes | ||
253 | * Runs user-defined Ansible playbooks | ||
254 | * Collects/reports results | ||
255 | * Potentially merges change | ||
261 | 256 | ||
262 | Gating | 257 | Gating |
263 | ====== | 258 | ====== |
@@ -269,22 +264,22 @@ Co-gating | |||
269 | ========= | 264 | ========= |
270 | 265 | ||
271 | .. cowsay:: Changes to a set of repositories merge monotonically such | 266 | .. cowsay:: Changes to a set of repositories merge monotonically such |
272 | that each change is tested with the current state of all | 267 | that each change is tested with the current state of all |
273 | other related repositories before it merges. | 268 | the other related repositories before it merges. |
274 | 269 | ||
275 | Parallel Co-gating | 270 | Parallel Co-gating |
276 | ================== | 271 | ================== |
277 | 272 | ||
278 | .. cowsay:: Changes are serialized such that each change is tested | 273 | .. cowsay:: Changes are serialized such that each change is tested |
279 | with all of the changes ahead of it to satisfy the | 274 | with all of the changes ahead of it to satisfy the |
280 | gating requirement while being able to run tests for | 275 | co-gating requirement while being able to run tests for |
281 | multiple changes simultaneously. | 276 | multiple changes simultaneously. |
282 | 277 | ||
283 | Presentation Checklist | 278 | Presentation Checklist |
284 | ====================== | 279 | ====================== |
285 | 280 | ||
286 | :: | 281 | :: |
287 | 282 | ||
288 | [x] Logos | 283 | [x] Logos |
289 | [x] Architecture diagram | 284 | [x] Architecture diagram |
290 | [x] Cows | 285 | [x] Cows |
@@ -294,9 +289,7 @@ Zuul Simulation | |||
294 | .. transition:: pan | 289 | .. transition:: pan |
295 | .. container:: handout | 290 | .. container:: handout |
296 | 291 | ||
297 | * That was a lot of words - let's walk through it one step at a time | 292 | * todo |
298 | * Here we have two git repos, called nova and keystone, and their | ||
299 | current HEAD state | ||
300 | 293 | ||
301 | .. ansi:: images/zsim-00.ans | 294 | .. ansi:: images/zsim-00.ans |
302 | 295 | ||
@@ -304,8 +297,8 @@ Zuul Simulation | |||
304 | =============== | 297 | =============== |
305 | .. transition:: cut | 298 | .. transition:: cut |
306 | .. container:: handout | 299 | .. container:: handout |
307 | 300 | ||
308 | * A change is approved for Nova | 301 | * todo |
309 | 302 | ||
310 | .. ansi:: images/zsim-01.ans | 303 | .. ansi:: images/zsim-01.ans |
311 | 304 | ||
@@ -314,9 +307,7 @@ Zuul Simulation | |||
314 | .. transition:: cut | 307 | .. transition:: cut |
315 | .. container:: handout | 308 | .. container:: handout |
316 | 309 | ||
317 | * Zuul starts running jobs for it | 310 | * todo |
318 | * The tests will test the current state of nova and keystone PLUS this nova | ||
319 | change | ||
320 | 311 | ||
321 | .. ansi:: images/zsim-02.ans | 312 | .. ansi:: images/zsim-02.ans |
322 | 313 | ||
@@ -325,7 +316,7 @@ Zuul Simulation | |||
325 | .. transition:: cut | 316 | .. transition:: cut |
326 | .. container:: handout | 317 | .. container:: handout |
327 | 318 | ||
328 | * A change is approved for Keystone | 319 | * todo |
329 | 320 | ||
330 | .. ansi:: images/zsim-03.ans | 321 | .. ansi:: images/zsim-03.ans |
331 | 322 | ||
@@ -334,8 +325,7 @@ Zuul Simulation | |||
334 | .. transition:: cut | 325 | .. transition:: cut |
335 | .. container:: handout | 326 | .. container:: handout |
336 | 327 | ||
337 | * The tests will test the current state of nova and keystone PLUS this nova | 328 | * todo |
338 | change | ||
339 | 329 | ||
340 | .. ansi:: images/zsim-04.ans | 330 | .. ansi:: images/zsim-04.ans |
341 | 331 | ||
@@ -501,159 +491,32 @@ Zuul Simulation | |||
501 | 491 | ||
502 | .. ansi:: images/zsim-22.ans | 492 | .. ansi:: images/zsim-22.ans |
503 | 493 | ||
504 | Cross-Project Problem | ||
505 | ===================== | ||
506 | |||
507 | * User reports bug in shade - auto_ip is not discovering their NAT properly | ||
508 | * Two fixes, one to detection algorithm, one to config override | ||
509 | * Config override requires adding support to os-client-config | ||
510 | * Once support is added to os-client-config, it can be consumed in shade | ||
511 | * How do we integration test this without releasing os-client-config? | ||
512 | |||
513 | Cross-Project Dependencies | 494 | Cross-Project Dependencies |
514 | ========================== | 495 | ========================== |
515 | 496 | ||
516 | Testing or gating dependencies (including jobs) manually specified by | 497 | Testing or gating dependencies manually specified by developers |
517 | developers | ||
518 | 498 | ||
519 | .. container:: progressive | 499 | .. container:: progressive |
520 | 500 | ||
521 | * shade https://review.openstack.org/#/c/513913/ | 501 | * shade https://review.openstack.org/513913 |
522 | 502 | ||
523 | Add unittest tips jobs | 503 | Add unittest tips jobs |
524 | 504 | * os-client-config https://review.openstack.org/513915 | |
525 | Change-ID: I5b411be5c5aa43535fa89a51d6099aadd7a8ea60 | ||
526 | * os-client-config https://review.openstack.org/#/c/513915 | ||
527 | 505 | ||
528 | Add shade-tox-tips jobs | 506 | Add shade-tox-tips jobs |
529 | 507 | ||
530 | Change-ID: Ie3e9a4deca1d74b94e810e87e130706fe15fe2c9 | 508 | Depends-On: https://review.openstack.org/513913 |
531 | 509 | * os-client-config https://review.openstack.org/513751 | |
532 | Depends-On: I5b411be5c5aa43535fa89a51d6099aadd7a8ea60 | ||
533 | * os-client-config https://review.openstack.org/#/c/513751/ | ||
534 | 510 | ||
535 | Added nat_source flag for networks | 511 | Added nat_source flag for networks |
536 | 512 | ||
537 | Change-ID: I3d8dd6d734a1013d2d4a43e11c3538c3a345820b | 513 | Depends-On: https://review.openstack.org/513915 |
538 | 514 | * shade https://review.openstack.org/51391 | |
539 | * shade https://review.openstack.org/#/c/513914 | ||
540 | 515 | ||
541 | Add support for configured NAT source variable | 516 | Add support for configured NAT source variable |
542 | 517 | ||
543 | Change-Id: I4b50c2323a487b5ce90f9d38a48be249cfb739c5 | 518 | Depends-On: https://review.openstack.org/513751 |
544 | |||
545 | Depends-On: I3d8dd6d734a1013d2d4a43e11c3538c3a345820b | ||
546 | |||
547 | shade: Add unittest tips jobs | ||
548 | ============================= | ||
549 | |||
550 | * In git.openstack.org/openstack-infra/shade/.zuul.yaml: | ||
551 | |||
552 | .. code:: yaml | ||
553 | |||
554 | - job: | ||
555 | name: shade-tox-py27-tips | ||
556 | parent: openstack-tox-py27 | ||
557 | description: | | ||
558 | Run tox python 27 unittests against master of important libs | ||
559 | required-projects: | ||
560 | - openstack-infra/shade | ||
561 | - openstack/keystoneauth | ||
562 | - openstack/os-client-config | ||
563 | |||
564 | - job: | ||
565 | name: shade-tox-py35-tips | ||
566 | parent: openstack-tox-py35 | ||
567 | description: | | ||
568 | Run tox python 35 unittests against master of important libs | ||
569 | required-projects: | ||
570 | - openstack-infra/shade | ||
571 | - openstack/keystoneauth | ||
572 | - openstack/os-client-config | ||
573 | |||
574 | shade: Add unittest tips project-template | ||
575 | ========================================= | ||
576 | |||
577 | * In git.openstack.org/openstack-infra/shade/.zuul.yaml: | ||
578 | |||
579 | .. code:: yaml | ||
580 | |||
581 | - project-template: | ||
582 | name: shade-tox-tips | ||
583 | check: | ||
584 | jobs: | ||
585 | - shade-tox-py27-tips | ||
586 | - shade-tox-py35-tips | ||
587 | gate: | ||
588 | jobs: | ||
589 | - shade-tox-py27-tips | ||
590 | - shade-tox-py35-tips | ||
591 | |||
592 | shade: Add unittest tips project-template to project | ||
593 | ==================================================== | ||
594 | |||
595 | * In git.openstack.org/openstack-infra/shade/.zuul.yaml: | ||
596 | |||
597 | .. code:: yaml | ||
598 | |||
599 | - project: | ||
600 | templates: | ||
601 | - publish-to-pypi | ||
602 | - publish-openstack-sphinx-docs | ||
603 | - shade-tox-tips | ||
604 | |||
605 | os-client-config: Add shade-tox-tips jobs | ||
606 | ========================================= | ||
607 | |||
608 | * In git.openstack.org/openstack/os-client-config/.zuul.yaml: | ||
609 | |||
610 | .. code:: yaml | ||
611 | 519 | ||
612 | - project: | ||
613 | templates: | ||
614 | - shade-tox-tips | ||
615 | check: | ||
616 | jobs: | ||
617 | - legacy-osc-dsvm-functional-tips: | ||
618 | voting: false | ||
619 | |||
620 | os-client-config: Add nat_source flag for networks | ||
621 | ================================================== | ||
622 | |||
623 | :: | ||
624 | |||
625 | diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py | ||
626 | index 2e97629..d1a6983 100644 | ||
627 | --- a/os_client_config/cloud_config.py | ||
628 | +++ b/os_client_config/cloud_config.py | ||
629 | @@ -581,3 +581,10 @@ class CloudConfig(object): | ||
630 | if net['nat_destination']: | ||
631 | return net['name'] | ||
632 | return None | ||
633 | + | ||
634 | + def get_nat_source(self): | ||
635 | + """Get network used for NAT source.""" | ||
636 | + for net in self.config['networks']: | ||
637 | + if net.get('nat_source'): | ||
638 | + return net['name'] | ||
639 | + return None | ||
640 | |||
641 | shade: Add support for configured NAT source variable | ||
642 | ===================================================== | ||
643 | |||
644 | :: | ||
645 | |||
646 | Zuul 10-21 13:57 | ||
647 | Patch Set 5: Verified-1 | ||
648 | Build failed. | ||
649 | openstack-tox-pep8 SUCCESS in 2m 29s | ||
650 | openstack-tox-py27 FAILURE in 2m 34s | ||
651 | build-openstack-releasenotes SUCCESS in 2m 47s | ||
652 | openstack-tox-py35 FAILURE in 2m 41s | ||
653 | openstack-tox-cover POST_FAILURE in 3m 52s (non-voting) | ||
654 | build-openstack-sphinx-docs SUCCESS in 2m 57s | ||
655 | shade-tox-py27-tips SUCCESS in 3m 18s | ||
656 | shade-tox-py35-tips SUCCESS in 2m 28s | ||
657 | 520 | ||
658 | Live Configuration Changes | 521 | Live Configuration Changes |
659 | ========================== | 522 | ========================== |
@@ -668,16 +531,12 @@ Live Configuration Changes | |||
668 | name: openstack | 531 | name: openstack |
669 | source: | 532 | source: |
670 | gerrit: | 533 | gerrit: |
671 | config-projects: | 534 | config-repos: |
672 | - project-config | 535 | - 'project-config' |
673 | untrusted-projects: | 536 | project-repos: |
674 | - openstack-infra/zuul-jobs: | 537 | - 'nova' |
675 | shadow: openstack-infra/project-config | 538 | - 'keystone' |
676 | - openstack-infra/openstack-zuul-jobs | 539 | - 'devstack-gate' |
677 | - openstack-infra/nodepool | ||
678 | - openstack-infra/shade | ||
679 | - openstack-infra/zuul | ||
680 | - openstack/requirements | ||
681 | 540 | ||
682 | Zuul Startup | 541 | Zuul Startup |
683 | ============ | 542 | ============ |
@@ -697,10 +556,9 @@ Zuul Startup | |||
697 | 556 | ||
698 | * Read config file | 557 | * Read config file |
699 | * Ask mergers for branches of each repo | 558 | * Ask mergers for branches of each repo |
700 | * Ask mergers for .zuul.yaml file for each branch of each repo | 559 | * Ask mergers for .zuul.yaml for each branch |
701 | 560 | ||
702 | ``.zuul.yaml`` can be ``^\.?zuul.yaml$`` file or ``^\.?zuul.d$`` run-parts | 561 | of each repo |
703 | directory. | ||
704 | 562 | ||
705 | .. ansi:: images/startup2.ans | 563 | .. ansi:: images/startup2.ans |
706 | 564 | ||
@@ -713,9 +571,9 @@ When .zuul.yaml Changes | |||
713 | * Asks mergers for updated content | 571 | * Asks mergers for updated content |
714 | * Splices into configuration used for that change | 572 | * Splices into configuration used for that change |
715 | * Works with cross-repo dependencies | 573 | * Works with cross-repo dependencies |
716 | 574 | ||
717 | ("This change depends on a change to the job definition") | 575 | ("This change depends on a change to the job definition") |
718 | 576 | ||
719 | How do you use this thing? | 577 | How do you use this thing? |
720 | ========================== | 578 | ========================== |
721 | .. transition:: tilt | 579 | .. transition:: tilt |
@@ -725,11 +583,10 @@ How do you use this thing? | |||
725 | Pipelines | 583 | Pipelines |
726 | ========= | 584 | ========= |
727 | 585 | ||
728 | * Describes the process flow and lifecycle **for a change** | ||
729 | * A process definition that connects git repositories, jobs, and | 586 | * A process definition that connects git repositories, jobs, and |
730 | reporting mechanisms. | 587 | reporting mechanisms. |
731 | * A context to fix a set of jobs to each project. | 588 | * A context to fix a set of jobs to each project. |
732 | 589 | ||
733 | Check Pipeline | 590 | Check Pipeline |
734 | ============== | 591 | ============== |
735 | 592 | ||
@@ -742,7 +599,7 @@ Check Pipeline | |||
742 | trigger: | 599 | trigger: |
743 | gerrit: | 600 | gerrit: |
744 | - event: patchset-created | 601 | - event: patchset-created |
745 | - event: change-restored | 602 | - event: change-restored |
746 | success: | 603 | success: |
747 | gerrit: | 604 | gerrit: |
748 | verified: 1 | 605 | verified: 1 |
@@ -755,6 +612,7 @@ Gate Pipeline | |||
755 | - pipeline: | 612 | - pipeline: |
756 | name: gate | 613 | name: gate |
757 | manager: dependent | 614 | manager: dependent |
615 | source: gerrit | ||
758 | trigger: | 616 | trigger: |
759 | gerrit: | 617 | gerrit: |
760 | - event: comment-added | 618 | - event: comment-added |
@@ -765,153 +623,29 @@ Gate Pipeline | |||
765 | verified: 2 | 623 | verified: 2 |
766 | submit: true | 624 | submit: true |
767 | 625 | ||
768 | Zuul Github Support | ||
769 | =================== | ||
770 | |||
771 | .. code:: yaml | ||
772 | |||
773 | - pipeline: | ||
774 | name: check | ||
775 | manager: independent | ||
776 | trigger: | ||
777 | github: | ||
778 | - event: pull_request | ||
779 | action: | ||
780 | - opened | ||
781 | - changed | ||
782 | - reopened | ||
783 | success: | ||
784 | github: | ||
785 | status: 'success' | ||
786 | failure: | ||
787 | github: | ||
788 | status: 'failure' | ||
789 | |||
790 | OpenStack Github Support for Cross Community Testing | ||
791 | ==================================================== | ||
792 | |||
793 | * Github App "OpenStack Zuul" | ||
794 | * App added to github project by project admin | ||
795 | * Project aded to OpenStack's main.yaml | ||
796 | * Test interactions between OpenStack and important adjacent communities | ||
797 | * https://github.com/ansible/ansible/pull/20974 | ||
798 | |||
799 | Cross Community Testing | ||
800 | ======================= | ||
801 | |||
802 | .. code:: yaml | ||
803 | |||
804 | - pipeline: | ||
805 | name: check | ||
806 | description: | | ||
807 | Newly uploaded patchsets enter this pipeline to receive an | ||
808 | initial +/-1 Verified vote. | ||
809 | manager: independent | ||
810 | trigger: | ||
811 | gerrit: | ||
812 | - event: patchset-created | ||
813 | - event: change-restored | ||
814 | - event: comment-added | ||
815 | comment: (?i)^(Patch Set [0-9]+:)?( [\w\\+-]*)*(\n\n)?\s*recheck | ||
816 | - event: comment-added | ||
817 | require-approval: | ||
818 | - Verified: [-1, -2] | ||
819 | username: zuul | ||
820 | approval: | ||
821 | - Workflow: 1 | ||
822 | github: | ||
823 | - event: pull_request | ||
824 | action: | ||
825 | - opened | ||
826 | - changed | ||
827 | - reopened | ||
828 | - event: pull_request | ||
829 | action: comment | ||
830 | comment: (?i)^\s*recheck\s*$ | ||
831 | |||
832 | Cross Community Support cont. | ||
833 | ============================= | ||
834 | |||
835 | .. code:: yaml | ||
836 | |||
837 | start: | ||
838 | github: | ||
839 | status: pending | ||
840 | comment: false | ||
841 | success: | ||
842 | gerrit: | ||
843 | # Note that gerrit keywords are case-sensitive. | ||
844 | Verified: 1 | ||
845 | github: | ||
846 | status: 'success' | ||
847 | mysql: | ||
848 | failure: | ||
849 | gerrit: | ||
850 | Verified: -1 | ||
851 | github: | ||
852 | status: 'failure' | ||
853 | mysql: | ||
854 | |||
855 | Cross Source Dependencies | ||
856 | ========================= | ||
857 | |||
858 | .. container:: progressive | ||
859 | |||
860 | * shade https://review.openstack.org/539563 | ||
861 | |||
862 | Shift voting flag and test_matrix_branch for ansible-devel job | ||
863 | |||
864 | Change-ID: Ic9d3983de641dbe618c65b2cbf2dcfa3686575df | ||
865 | |||
866 | * ansible https://github.com/ansible/ansible/pull/34925 | ||
867 | |||
868 | continue fact gathering even without dmidecode | ||
869 | |||
870 | * ansible https://github.com/ansible/ansible/pull/20974 | ||
871 | |||
872 | Make a generalized OpenStack cloud constructor | ||
873 | |||
874 | Depends-On: https://review.openstack.org/539563 | ||
875 | Depends-On: https://github.com/ansible/ansible/pull/34925 | ||
876 | |||
877 | Jobs | 626 | Jobs |
878 | ==== | 627 | ==== |
879 | 628 | ||
880 | * Jobs run on nodes from nodepool (static or dynamic) | 629 | * Jobs run on nodes from nodepool (static or dynamic) |
881 | * Metadata defined in Zuul's configuration | 630 | * Metadata defined in Zuul's configuration |
882 | * Execution content in Ansible (with live streaming!) | 631 | * Execution content in Ansible |
883 | * Jobs may be defined centrally or in the repo being tested | 632 | * Jobs may be defined centrally or in the repo being tested |
884 | * Jobs have contextual variants that simplify configuration | 633 | * Jobs have contextual variants that simplify configuration |
885 | 634 | ||
886 | Shared Job Configs | ||
887 | ================== | ||
888 | |||
889 | * Job config repos are all in git | ||
890 | * Designed to support directly sharing job configurations | ||
891 | * git.openstack.org/openstack-infra/zuul-jobs repo is a 'standard library' | ||
892 | to be directly shared between zuul installations | ||
893 | |||
894 | Job | 635 | Job |
895 | === | 636 | === |
896 | 637 | ||
897 | .. code:: yaml | 638 | .. code:: yaml |
898 | 639 | ||
899 | - job: | 640 | - job: |
900 | name: base | 641 | name: 'base' |
901 | parent: null | 642 | timeout: '30m' |
902 | description: | | 643 | nodes: 'ubuntu-xenial' |
903 | The base job for Zuul. | 644 | workspace: '/opt/workspace' |
904 | timeout: 1800 | 645 | pre-run: |
905 | nodeset: | 646 | - 'setup-host' |
906 | nodes: | ||
907 | - name: primary | ||
908 | label: centos-7 | ||
909 | pre-run: playbooks/base/pre.yaml | ||
910 | post-run: | 647 | post-run: |
911 | - playbooks/base/post-ssh.yaml | 648 | - 'archive-logs' |
912 | - playbooks/base/post-logs.yaml | ||
913 | secrets: | ||
914 | - site_logs | ||
915 | 649 | ||
916 | Simple Job | 650 | Simple Job |
917 | ========== | 651 | ========== |
@@ -919,17 +653,8 @@ Simple Job | |||
919 | .. code:: yaml | 653 | .. code:: yaml |
920 | 654 | ||
921 | - job: | 655 | - job: |
922 | name: tox | 656 | name: 'python27' |
923 | pre-run: playbooks/setup-tox.yaml | 657 | parent: 'base' |
924 | run: playbooks/tox.yaml | ||
925 | post-run: playbooks/fetch-tox-output.yaml | ||
926 | |||
927 | - job: | ||
928 | name: tox-py27 | ||
929 | parent: tox | ||
930 | vars: | ||
931 | tox_envlist: py27 | ||
932 | |||
933 | 658 | ||
934 | Simple Job Variant | 659 | Simple Job Variant |
935 | ================== | 660 | ================== |
@@ -937,92 +662,45 @@ Simple Job Variant | |||
937 | .. code:: yaml | 662 | .. code:: yaml |
938 | 663 | ||
939 | - job: | 664 | - job: |
940 | name: tox-py27 | 665 | name: 'python27' |
941 | branches: stable/mitaka | 666 | branch: 'stable/mitaka' |
942 | nodeset: | 667 | nodes: 'ubuntu-trusty' |
943 | - name: ubuntu-trusty | 668 | |
944 | label: ubuntu-trusty | ||
945 | |||
946 | Nodesets for Multi-node Jobs | ||
947 | ============================ | ||
948 | |||
949 | .. code:: yaml | ||
950 | |||
951 | - nodeset: | ||
952 | name: ceph-cluster | ||
953 | nodes: | ||
954 | - name: controller | ||
955 | label: centos-7 | ||
956 | - name: compute1 | ||
957 | label: fedora-26 | ||
958 | - name: compute2 | ||
959 | label: fedora-26 | ||
960 | groups: | ||
961 | - name: ceph-osd | ||
962 | nodes: | ||
963 | - controller | ||
964 | - name: ceph-monitor | ||
965 | nodes: | ||
966 | - controller | ||
967 | - compute1 | ||
968 | - compute2 | ||
969 | |||
970 | Multi-node Job | 669 | Multi-node Job |
971 | ============== | 670 | ============== |
671 | .. container:: handout | ||
972 | 672 | ||
973 | * nodesets are provided to Ansible for jobs in inventory | 673 | nodepool, shrews |
974 | 674 | ||
975 | .. code:: yaml | 675 | .. code:: yaml |
976 | 676 | ||
977 | - job: | 677 | - job: |
978 | name: ceph-multinode | 678 | name: 'devstack-multinode' |
979 | nodeset: ceph-cluster | 679 | parent: 'base' |
980 | run: playbooks/install-ceph.yaml | 680 | nodes: |
981 | 681 | - name: 'controller' | |
982 | Multi-node Ceph Job Content | 682 | image: 'ubuntu-xenial' |
983 | =========================== | 683 | - name: 'compute' |
984 | 684 | image: 'ubuntu-xenial' | |
985 | .. code:: yaml | ||
986 | |||
987 | - hosts: all | ||
988 | roles: | ||
989 | - install-ceph | ||
990 | - hosts: ceph-osd | ||
991 | roles: | ||
992 | - start-ceph-osd | ||
993 | - hosts: ceph-monitor | ||
994 | roles: | ||
995 | - start-ceph-monitor | ||
996 | - hosts: all | ||
997 | roles: | ||
998 | - do-something-interesting | ||
999 | 685 | ||
1000 | Projects | 686 | Projects |
1001 | ======== | 687 | ======== |
1002 | 688 | ||
1003 | * Projects are git repositories | 689 | * Projects are git repositories |
1004 | * Specify a set of jobs for each pipeline | 690 | * Specify a set of jobs for each pipeline |
1005 | * golang git repo naming as been adopted: | 691 | |
1006 | |||
1007 | :: | ||
1008 | |||
1009 | zuul@ubuntu-xenial:~$ find /home/zuul/src -mindepth 3 -maxdepth 3 -type d | ||
1010 | /home/zuul/src/git.openstack.org/openstack-infra/shade | ||
1011 | /home/zuul/src/git.openstack.org/openstack/keystoneauth | ||
1012 | /home/zuul/src/git.openstack.org/openstack/os-client-config | ||
1013 | /home/zuul/src/github.com/ansible/ansible | ||
1014 | |||
1015 | Project | 692 | Project |
1016 | ======= | 693 | ======= |
1017 | 694 | ||
1018 | .. code:: yaml | 695 | .. code:: yaml |
1019 | 696 | ||
1020 | - project: | 697 | - project: |
698 | name: 'nova' | ||
1021 | check: | 699 | check: |
1022 | jobs: | 700 | jobs: |
1023 | - openstack-tox-py27 | 701 | - python27 |
1024 | - openstack-tox-py35 | 702 | - python35 |
1025 | - openstack-doc-build | 703 | - docs |
1026 | 704 | ||
1027 | Project with Local Variant | 705 | Project with Local Variant |
1028 | ========================== | 706 | ========================== |
@@ -1030,13 +708,14 @@ Project with Local Variant | |||
1030 | .. code:: yaml | 708 | .. code:: yaml |
1031 | 709 | ||
1032 | - project: | 710 | - project: |
711 | name: 'nova' | ||
1033 | check: | 712 | check: |
1034 | jobs: | 713 | jobs: |
1035 | - openstack-tox-py27 | 714 | - python27 |
1036 | - openstack-tox-py35 | 715 | - python35 |
1037 | - openstack-doc-build | 716 | - docs |
1038 | - openstack-tox-pypy: | 717 | - pypy: |
1039 | voting: false | 718 | voting: false |
1040 | 719 | ||
1041 | Project with More Local Variants | 720 | Project with More Local Variants |
1042 | ================================ | 721 | ================================ |
@@ -1044,15 +723,15 @@ Project with More Local Variants | |||
1044 | .. code:: yaml | 723 | .. code:: yaml |
1045 | 724 | ||
1046 | - project: | 725 | - project: |
1047 | name: openstack/nova | 726 | name: 'nova' |
1048 | check: | 727 | check: |
1049 | jobs: | 728 | jobs: |
1050 | - openstack-tox-py27 | 729 | - python27 |
1051 | - openstack-tox-py35 | 730 | - python35 |
1052 | - openstack-doc-build: | 731 | - docs: |
1053 | files: '^docs/.*$' | 732 | files: '^docs/.*$' |
1054 | - openstack-tox-pypy: | 733 | - pypy: |
1055 | voting: false | 734 | voting: false |
1056 | 735 | ||
1057 | Project with Many Local Variants | 736 | Project with Many Local Variants |
1058 | ================================ | 737 | ================================ |
@@ -1060,106 +739,41 @@ Project with Many Local Variants | |||
1060 | .. code:: yaml | 739 | .. code:: yaml |
1061 | 740 | ||
1062 | - project: | 741 | - project: |
1063 | name: openstack/nova | 742 | name: 'nova' |
1064 | check: | 743 | check: |
1065 | jobs: | 744 | jobs: |
1066 | - openstack-tox-py27 | 745 | - python27: |
1067 | nodeset: | 746 | nodes: 'ubuntu-xenial' |
1068 | - name: centos-7 | 747 | - python27: |
1069 | label: centos-7 | 748 | branch: 'stable/newton' |
1070 | - openstack-tox-py27 | 749 | nodes: 'ubuntu-trusty' |
1071 | branches: stable/newton | 750 | - python35 |
1072 | nodeset: | 751 | - docs: |
1073 | - name: ubuntu-trusty | 752 | files: '^docs/.*$' |
1074 | label: ubuntu-trusty | 753 | - pypy: |
1075 | - openstack-doc-build: | 754 | voting: false |
1076 | files: '^docs/.*$' | ||
1077 | - openstack-tox-pypy: | ||
1078 | voting: false | ||
1079 | |||
1080 | Project With Central and Local Config | ||
1081 | ===================================== | ||
1082 | |||
1083 | .. code:: yaml | ||
1084 | |||
1085 | # In git.openstack.org/openstack-infra/project-config: | ||
1086 | - project: | ||
1087 | name: openstack/nova | ||
1088 | templates: | ||
1089 | - openstack-tox-jobs | ||
1090 | |||
1091 | .. code:: yaml | ||
1092 | |||
1093 | # In git.openstack.org/openstack/nova/.zuul.yaml: | ||
1094 | - project: | ||
1095 | check: | ||
1096 | - nova-placement-functional-devstack | ||
1097 | 755 | ||
1098 | Project with Job Dependencies | 756 | Project with Job Dependencies |
1099 | ============================= | 757 | ============================= |
1100 | 758 | ||
1101 | .. code:: yaml | 759 | .. code:: yaml |
1102 | 760 | ||
1103 | # In git.openstack.org/openstack-infra/project-config: | ||
1104 | - project: | 761 | - project: |
1105 | name: openstack/nova | 762 | name: nova |
1106 | release: | 763 | release: |
1107 | jobs: | 764 | jobs: |
1108 | - build-artifacts | 765 | - build-tarball: |
1109 | - upload-tarball: | 766 | jobs: |
1110 | dependencies: build-artifacts | 767 | - upload-tarball: |
1111 | - upload-pypi: | 768 | jobs: |
1112 | dependencies: build-artifacts | 769 | - update-mirror |
1113 | - notify-mirror: | ||
1114 | dependencies: | ||
1115 | - upload-tarball | ||
1116 | - upload-pypi | ||
1117 | 770 | ||
1118 | Playbooks | 771 | Playbooks |
1119 | ========= | 772 | ========= |
1120 | 773 | ||
1121 | * Jobs run Ansible playbooks | 774 | * Jobs run playbooks |
1122 | * Playbooks may be defined centrally or in the repo being tested | 775 | * Playbooks may be defined centrally or in the repo being tested |
1123 | * Playbooks can use roles from current or other Zuul repos | 776 | * Playbooks can use roles from current or other Zuul repos or Galaxy |
1124 | (or Galaxy, coming soon) | ||
1125 | * Playbooks are run on the zuul-executor using bubblewrap | ||
1126 | https://github.com/projectatomic/bubblewrap | ||
1127 | * Playbooks are not allowed to execute content on 'localhost' | ||
1128 | |||
1129 | Job with Roles | ||
1130 | ============== | ||
1131 | |||
1132 | .. code:: yaml | ||
1133 | |||
1134 | - job: | ||
1135 | name: zuul-integration | ||
1136 | description: | | ||
1137 | Multi-node Zuul installation and integration test | ||
1138 | nodeset: zuul-cluster | ||
1139 | roles: | ||
1140 | - zuul: openstack-infra/ansible-role-zuul | ||
1141 | run: playbooks/zuul-integration | ||
1142 | |||
1143 | Job with Multiple Projects | ||
1144 | ========================== | ||
1145 | |||
1146 | .. code:: yaml | ||
1147 | |||
1148 | - job: | ||
1149 | name: tox-py35-on-zuul | ||
1150 | parent: tox-py35 | ||
1151 | description: | | ||
1152 | Run zuul's py35 unittests on patches to zuul-jobs | ||
1153 | vars: | ||
1154 | zuul_work_dir: src/git.openstack.org/openstack-infra/zuul | ||
1155 | required-projects: | ||
1156 | - openstack-infra/zuul | ||
1157 | |||
1158 | - project: | ||
1159 | name: openstack-infra/zuul-jobs | ||
1160 | check: | ||
1161 | jobs: | ||
1162 | - tox-py35-on-zuul | ||
1163 | 777 | ||
1164 | Devstack-gate / Tempest Playbook | 778 | Devstack-gate / Tempest Playbook |
1165 | ================================ | 779 | ================================ |
@@ -1167,164 +781,42 @@ Devstack-gate / Tempest Playbook | |||
1167 | .. code:: yaml | 781 | .. code:: yaml |
1168 | 782 | ||
1169 | # devstack-gate / tempest playbook | 783 | # devstack-gate / tempest playbook |
784 | --- | ||
1170 | hosts: all | 785 | hosts: all |
1171 | roles: | 786 | roles: |
1172 | - setup-multinode-networking | 787 | - setup-multinode-networking |
1173 | - partition-swap | 788 | - partition-swap |
1174 | - configure-mirrors | 789 | - configure-mirrors |
1175 | - run-devstack | 790 | - run-devstack |
1176 | - run-tempest | 791 | - run-tempest |
1177 | 792 | ||
1178 | Simple Shell Playbook | 793 | Simple Shell Playbook |
1179 | ===================== | 794 | ===================== |
1180 | 795 | ||
1181 | .. code:: yaml | 796 | .. code:: yaml |
1182 | 797 | ||
798 | --- | ||
1183 | hosts: controller | 799 | hosts: controller |
1184 | tasks: | 800 | roles: |
1185 | - shell: ./run_tests.sh | 801 | - shell: | |
802 | cd $WORKSPACE | ||
803 | ./run_tests.sh | ||
804 | |||
1186 | 805 | ||
1187 | Test Like Production | 806 | Test Like Production |
1188 | ==================== | 807 | ==================== |
1189 | 808 | ||
1190 | If you use Ansible for deployment, your test and deployment processes | 809 | If you use Ansible for deployment, your test and deployment processes |
1191 | and playbooks are the same | 810 | and playbooks are the same |
1192 | 811 | ||
1193 | What if you don't use Ansible? | ||
1194 | ============================== | ||
1195 | |||
1196 | OpenStack Infra Control Plane uses Puppet | ||
1197 | ========================================= | ||
1198 | |||
1199 | .. code:: yaml | ||
1200 | |||
1201 | # In git.openstack.org/openstack-infra/project-config/roles/legacy-install-afs-with-puppet/tasks/main.yaml | ||
1202 | - name: Install puppet | ||
1203 | shell: ./install_puppet.sh | ||
1204 | args: | ||
1205 | chdir: "{{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/system-config" | ||
1206 | environment: | ||
1207 | # Skip setting up pip, our images have already done this. | ||
1208 | SETUP_PIP: "false" | ||
1209 | become: yes | ||
1210 | |||
1211 | - name: Copy manifest | ||
1212 | copy: | ||
1213 | src: manifest.pp | ||
1214 | dest: "{{ ansible_user_dir }}/manifest.pp" | ||
1215 | |||
1216 | - name: Run puppet | ||
1217 | puppet: | ||
1218 | manifest: "{{ ansible_user_dir }}/manifest.pp" | ||
1219 | become: yes | ||
1220 | |||
1221 | Secrets | ||
1222 | ======= | ||
1223 | |||
1224 | * Inspired by Kubernetes Secrets API | ||
1225 | * Projects can add named encrypted secrets to their .zuul.yaml file | ||
1226 | * Jobs can request to use secrets by name | ||
1227 | * Jobs using secrets are not reconfigured speculatively | ||
1228 | * Secrets can only be used by the same project they are defined in | ||
1229 | * Public key per project: | ||
1230 | ``{{ zuul_url }}/{{ tenant }}/{{ project }}.pub`` | ||
1231 | |||
1232 | :: | ||
1233 | GET http://zuul.openstack.org/openstack-infra/shade.pub | ||
1234 | |||
1235 | Secret Example (note, no admins had to enable this) | ||
1236 | =================================================== | ||
1237 | |||
1238 | .. code:: yaml | ||
1239 | |||
1240 | # In git.openstack.org/openstack/loci/.zuul.yaml: | ||
1241 | - secret: | ||
1242 | name: loci_docker_login | ||
1243 | data: | ||
1244 | user: !encrypted/pkcs1-oaep | ||
1245 | - r8Nbpq5olmfLF035BZ/CUoFLIdhvBi/49KuochOAHbvns+xMiho3C7MEFzYDqJX3IhHde | ||
1246 | BICYOgK7qnyINOIZL2e7pl75rEdHQwJjSFUMkpdY6wEP7f9hpolj9xVp0ifHUVQqPHMRn | ||
1247 | zoPFd8MEAHxH5GLmc2SWJ98E/QUqGltxBi1YRSZoCcNtq3tHFK5Y+xQlLhIseJ2HkpDs6 | ||
1248 | YXOGP9Qt4Va6sdyBcA90H+apSAcYA3Duu962ySZQAsYNui/3NQq3gLA+OZeyTJtcrh4hj | ||
1249 | Rb5dBnDWfSrMpxdNkbPXXgbQaxO3T0L4jbaOF8VKEsiI9olBrOeV2M9ddYJjSsHGj4XR8 | ||
1250 | 4vwS0+doB7np93fujiDuHVgdG8R40NW2GznyKRlRtzAORla7Mzw1Y1MokcUyY6p1LlLLl | ||
1251 | wUuWYCCEuRciOPhZXQ2u42qju/zrK2/dPnO8HfUINSrN0WbNq14ZwPpbj0ro02oGPbtwu | ||
1252 | OTw1z+N0Nc+GuLWlwYJGYM/z0UnvDR3WEBc2kXbVev9w4n0cB3RyphML2PDZZWbw8tjnX | ||
1253 | h1VsAOJ0Qo4qq1K/ft95ypd+vtjkfepEgHEBmJNwutJa9IHAkGfrkO9VkpUTPpfffnPwz | ||
1254 | d0/zaaadNl6MLQUSutRwY23YIIbv+fmukxw2vnJmvn6abkBlMya7KgtifwNA8c= | ||
1255 | password: !encrypted/pkcs1-oaep | ||
1256 | - gUEX4eY3JAk/Xt7Evmf/hF7xr6HpNRXTibZjrKTbmI4QYHlzEBrBbHey27Pt/eYvKKeKw | ||
1257 | hk8MDQ4rNX7ZK1v+CKTilUfOf4AkKYbe6JFDd4z+zIZ2PAA7ZedO5FY/OnqrG7nhLvQHE | ||
1258 | 5nQrYwmxRp4O8eU5qG1dSrM9X+bzri8UnsI7URjqmEsIvlUqtybQKB9qQXT4d6mOeaKGE | ||
1259 | 5h6Ydkb9Zdi4Qh+GpCGDYwHZKu1mBgVK5M1G6NFMy1DYz+4NJNkTRe9J+0TmWhQ/KZSqo | ||
1260 | 4ck0x7Tb0Nr7hQzV8SxlwkaCTLDzvbiqmsJPLmzXY2jry6QsaRCpthS01vnj47itoZ/7p | ||
1261 | taH9CoJ0Gl7AkaxsrDSVjWSjatTQpsy1ub2fuzWHH4ASJFCiu83Lb2xwYts++r8ZSn+mA | ||
1262 | hbEs0GzPI6dIWg0u7aUsRWMOB4A+6t2IOJibVYwmwkG8TjHRXxVCLH5sY+i3MR+NicR9T | ||
1263 | IZFdY/AyH6vt5uHLQDU35+5n91pUG3F2lyiY5aeMOvBL05p27GTMuixR5ZoHcvSoHHtCq | ||
1264 | 7Wnk21iHqmv/UnEzqUfXZOque9YP386RBWkshrHd0x3OHUfBK/WrpivxvIGBzGwMr2qAj | ||
1265 | /AhJsfDXKBBbhGOGk1u5oBLjeC4SRnAcIVh1+RWzR4/cAhOuy2EcbzxaGb6VTM= | ||
1266 | |||
1267 | Secret Example | ||
1268 | ============== | ||
1269 | |||
1270 | .. code:: yaml | ||
1271 | |||
1272 | # In git.openstack.org/openstack/loci/.zuul.yaml: | ||
1273 | - job: | ||
1274 | name: publish-loci-cinder | ||
1275 | parent: loci-cinder | ||
1276 | post-run: playbooks/push | ||
1277 | secrets: | ||
1278 | - loci_docker_login | ||
1279 | |||
1280 | # In git.openstack.org/openstack/loci/playbooks/push.yaml: | ||
1281 | - hosts: all | ||
1282 | tasks: | ||
1283 | - include_vars: vars.yaml | ||
1284 | |||
1285 | - name: Push project to DockerHub | ||
1286 | block: | ||
1287 | - command: docker login -u {{ loci_docker_login.user }} -p {{ loci_docker_login.password }} | ||
1288 | no_log: True | ||
1289 | - command: docker push openstackloci/{{ project }}:{{ branch }}-{{ item.name }} | ||
1290 | with_items: "{{ distros }}" | ||
1291 | |||
1292 | Status | ||
1293 | ====== | ||
1294 | |||
1295 | * Zuul v3 is in production for OpenStack (in OpenStack VMs) | ||
1296 | * Zuul v3 also runing at BMW (in OpenShift) and Godaddy (in kuberenetes) | ||
1297 | and Huawei's OpenLab. | ||
1298 | * Software Factory updated to v3 | ||
1299 | https://softwarefactory-project.io/sf/welcome.html | ||
1300 | * will tag and release v3.0 once we're satisfied it's good for other people | ||
1301 | (within the next few weeks) | ||
1302 | |||
1303 | What's Next? | ||
1304 | ============ | ||
1305 | |||
1306 | * shared job doc generation | ||
1307 | * node providers | ||
1308 | * kuberenetes | ||
1309 | * OCI/docker | ||
1310 | * Mac Stadium (for our Ansible friends) | ||
1311 | * ec2 | ||
1312 | * ansible | ||
1313 | * support for galaxy roles | ||
1314 | * in-line code-review comments from Zuul | ||
1315 | * native container/kubernetes job execution | ||
1316 | |||
1317 | Important Links | 812 | Important Links |
1318 | =============== | 813 | =============== |
1319 | 814 | ||
1320 | * https://zuul-ci.org/ | 815 | * https://zuul-ci.org/ |
1321 | * https://git.openstack.org/cgit/openstack-infra/zuul | 816 | * https://git.zuul-ci.org/cgit/zuul |
1322 | * https://docs.openstack.org/infra/zuul | 817 | * https://zuul-ci.org/docs/zuul |
1323 | * https://docs.openstack.org/infra/manual/zuulv3.html | 818 | * https://zuul-ci.org/docs/zuul-jobs/ |
1324 | * https://docs.openstack.org/infra/zuul-jobs/ | ||
1325 | * https://docs.openstack.org/infra/openstack-zuul-jobs/ | 819 | * https://docs.openstack.org/infra/openstack-zuul-jobs/ |
1326 | * https://storyboard.openstack.org/#!/project/679 | ||
1327 | * https://storyboard.openstack.org/#!/board/41 | ||
1328 | * freenode:#zuul | 820 | * freenode:#zuul |
1329 | 821 | ||
1330 | Questions | 822 | Questions |