diff options
Diffstat (limited to 'src/thousands/thousands.rst')
-rw-r--r-- | src/thousands/thousands.rst | 1001 |
1 files changed, 1001 insertions, 0 deletions
diff --git a/src/thousands/thousands.rst b/src/thousands/thousands.rst new file mode 100644 index 0000000..ec20202 --- /dev/null +++ b/src/thousands/thousands.rst | |||
@@ -0,0 +1,1001 @@ | |||
1 | . display in 68x24 | ||
2 | .. display in 88x24 | ||
3 | |||
4 | .. pygments yaml? (only file breaks (---) tinted) | ||
5 | .. slide on high level v3 changes | ||
6 | .. slide on nodepool | ||
7 | |||
8 | .. transition:: dissolve | ||
9 | :duration: 0.4 | ||
10 | |||
11 | Test Slide | ||
12 | ========== | ||
13 | .. hidetitle:: | ||
14 | |||
15 | .. ansi:: images/testslide.ans | ||
16 | |||
17 | Testing Thousands of Python Projects Every Day | ||
18 | ============================================== | ||
19 | |||
20 | Red Hat | ||
21 | ======= | ||
22 | |||
23 | .. hidetitle:: | ||
24 | .. container:: handout | ||
25 | |||
26 | * I work for Red Hat in the CTO Office as the Chief Architect | ||
27 | for CI/CD | ||
28 | |||
29 | .. ansi:: images/redhat.ans | ||
30 | |||
31 | OpenStack | ||
32 | ========= | ||
33 | |||
34 | .. container:: handout | ||
35 | |||
36 | * I work on OpenStack. | ||
37 | * I sit on the Board of Directors. I was on the Technical Committee | ||
38 | |||
39 | .. hidetitle:: | ||
40 | .. ansi:: images/openstack.ans | ||
41 | |||
42 | OpenStack Infra | ||
43 | =============== | ||
44 | |||
45 | .. container:: handout | ||
46 | |||
47 | * My primary technical role with OpenStack is working on the OpenStack CI | ||
48 | system. | ||
49 | |||
50 | :: | ||
51 | |||
52 | "most insane CI infrastructure I've ever been a part of" | ||
53 | |||
54 | -- Alex Gaynor | ||
55 | |||
56 | "OpenStack Infra are like the SpaceX of CI" | ||
57 | |||
58 | -- Emily Dunham | ||
59 | |||
60 | tl;dr | ||
61 | ===== | ||
62 | |||
63 | * multi repo | ||
64 | * integrated deliverable | ||
65 | * gated commits | ||
66 | * open tooling | ||
67 | * nobody is special | ||
68 | * there is no Dana, only Zuul | ||
69 | |||
70 | OpenStack Is | ||
71 | ============ | ||
72 | |||
73 | * Federated | ||
74 | * Distributed | ||
75 | * Large | ||
76 | * Open | ||
77 | * Not Alone | ||
78 | |||
79 | Federated | ||
80 | ========= | ||
81 | |||
82 | * Hundreds of involved companies | ||
83 | * No 'main' company | ||
84 | * "Decisions are made by those who show up" | ||
85 | * Union of priorities/use cases | ||
86 | |||
87 | Impact of being Federated | ||
88 | ========================= | ||
89 | |||
90 | * No company can appoint humans to project positions | ||
91 | * The project cannot fire anyone | ||
92 | * Variable background of contributors | ||
93 | * Heavy reliance on consensus-oriented democracy | ||
94 | |||
95 | Distributed | ||
96 | =========== | ||
97 | |||
98 | * There is no office | ||
99 | * Contributor base is global | ||
100 | * Multitude of contributor backgrounds | ||
101 | |||
102 | Impact of being Distributed | ||
103 | =========================== | ||
104 | |||
105 | * Constantly at odds with American Exceptionalism | ||
106 | * Tooling must empower all contributors, regardless of background, | ||
107 | skill level or cultural context | ||
108 | * Heavy preference for text-based communication | ||
109 | * Cannot assume US-centric needs or solutions | ||
110 | |||
111 | Large numbers of | ||
112 | ================ | ||
113 | |||
114 | * Contributors (\~2k in any given 6 month period) | ||
115 | * Changes | ||
116 | * Code Repositories (1904 as of this morning) | ||
117 | |||
118 | OpenStack Scale Comparison | ||
119 | ========================== | ||
120 | |||
121 | * 2KJPH (2,000 jobs per hour) | ||
122 | * Build Nodes from 13 Regions of 5 Public and 2 Private OpenStack Clouds | ||
123 | * Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro, Limestone | ||
124 | * 10,000 changes merged per month | ||
125 | |||
126 | OpenStack Scale Comparison | ||
127 | ========================== | ||
128 | |||
129 | * 2KJPH (2,000 jobs per hour) | ||
130 | * Nodes from 12 Regions of 5 Public and 1 Private OpenStack Clouds | ||
131 | * Rackspace, Internap, OVH, Vexxhost, CityCloud and Linaro, Limestone | ||
132 | * 10,000 changes merged per month | ||
133 | |||
134 | By comparison, our friends at the amazing project Ansible received | ||
135 | 13,000 changes and had merged 8,000 of them in its first 4 years. | ||
136 | |||
137 | Four Opens | ||
138 | ========== | ||
139 | |||
140 | * Open Source | ||
141 | (we don't hold back Enterprise features, we don't cripple things) | ||
142 | * Open Design | ||
143 | (design process open to all, decisions are not made inside company doors) | ||
144 | * Open Development | ||
145 | (public source code, public code review, all code is reviewed and gated) | ||
146 | * Open Community | ||
147 | (lazy consensus, democratic leadership from participants, | ||
148 | public logged meetings in IRC, public archived mailing lists) | ||
149 | |||
150 | Nobody is Special | ||
151 | ================= | ||
152 | |||
153 | * No dictators | ||
154 | * Aggressively egalitarian | ||
155 | * No "pay for play" | ||
156 | |||
157 | Fifth Open - Four Opens Applied to the Infrastructure | ||
158 | ===================================================== | ||
159 | |||
160 | * All tools must be Open Source | ||
161 | * Any external services must by Open Source | ||
162 | * Strongly avoid single-vendor | ||
163 | |||
164 | All Tools are Open Source | ||
165 | ========================= | ||
166 | |||
167 | .. hidetitle:: | ||
168 | |||
169 | No Developer is ever be required to use a proprietary tool | ||
170 | to work on OpenStack. | ||
171 | |||
172 | Sixth Open - Four Opens Applied to Operations | ||
173 | ============================================= | ||
174 | |||
175 | * Ops driven by git/code-review - not by humans running commands | ||
176 | * Run as many things CD as possible | ||
177 | * Infrastructure team operates the same as the project | ||
178 | * Core reviewer status and root access are earned | ||
179 | * Human-initiated ops actions (running commands, clicking a UI) are a bug | ||
180 | * Keys/secrets are not Open :) | ||
181 | |||
182 | We're Not Alone | ||
183 | =============== | ||
184 | |||
185 | * Dependencies (libvirt/kvm/xen, mysql/pg, rabbit, | ||
186 | python/javascript, ceph/gluster, ansible/salt/puppet/chef, ovs/odl) | ||
187 | * Adjacencies (kubernetes, ansible, terraform, opnfv, spinnaker) | ||
188 | * Vendors (plugins, products, services, distros) | ||
189 | |||
190 | Lessons from a massive, openly-developed project | ||
191 | ================================================ | ||
192 | |||
193 | Thierry Carrez is speaking at 14:45 today with more on this. | ||
194 | |||
195 | Developer Process In a Nutshell | ||
196 | =============================== | ||
197 | |||
198 | * Code Review - nobody has direct commit/push access | ||
199 | * 3rd-Party CI for vendors | ||
200 | * Gated Commits | ||
201 | |||
202 | Gated Commits | ||
203 | ============= | ||
204 | |||
205 | :: | ||
206 | |||
207 | Hack Review Test | ||
208 | ========= ========== ========== | ||
209 | |||
210 | push approve | ||
211 | +-------------+ +-------------+ | ||
212 | | | | | | ||
213 | +------+--+ +--v----+--+ +--v-------+ | ||
214 | | | | | | | | ||
215 | | $EDITOR | | Gerrit | | Zuul | | ||
216 | | | | | | | | ||
217 | +------^--+ +--+----^--+ +--+-------+ | ||
218 | | | | | | ||
219 | +-------------+ +-------------+ | ||
220 | clone merge | ||
221 | |||
222 | Gating | ||
223 | ====== | ||
224 | |||
225 | Every change proposed for a repository is tested before it merges. | ||
226 | |||
227 | Co-gating | ||
228 | ========= | ||
229 | |||
230 | Changes to a set of repositories merge monotonically such | ||
231 | that each change is tested with the current state of all | ||
232 | other related repositories before it merges. | ||
233 | |||
234 | Parallel Co-gating | ||
235 | ================== | ||
236 | |||
237 | Changes are serialized such that each change is tested | ||
238 | with all of the changes ahead of it to satisfy the | ||
239 | gating requirement while being able to run tests for | ||
240 | multiple changes simultaneously. | ||
241 | |||
242 | Zuul | ||
243 | ==== | ||
244 | |||
245 | * Custom-written multi-repo parallel co-gating engine | ||
246 | * When to run | ||
247 | * Where to run it | ||
248 | * With what git states | ||
249 | * How to respond to results | ||
250 | |||
251 | Zuul Architecture | ||
252 | ================= | ||
253 | |||
254 | .. ansi:: images/architecture.ans | ||
255 | |||
256 | Zuul is written in Python 3.5 | ||
257 | ============================= | ||
258 | |||
259 | Not just for OpenStack | ||
260 | ====================== | ||
261 | |||
262 | * Zuul v3 is in production for OpenStack (in OpenStack VMs) | ||
263 | |||
264 | Also running at: | ||
265 | |||
266 | * BMW (control plane in OpenShift) | ||
267 | * Godaddy (control plane in Kubernetes) | ||
268 | * OpenContrail | ||
269 | * OpenLab | ||
270 | * others ... | ||
271 | |||
272 | Zuul is not a general purpose automation framework | ||
273 | ================================================== | ||
274 | |||
275 | Zuul in a nutshell | ||
276 | ================== | ||
277 | |||
278 | * Listens for code events | ||
279 | * Prepares appropriate job config and git repo states | ||
280 | * Allocates nodes for test jobs | ||
281 | * Pushes git repo states to nodes | ||
282 | * Runs user-defined Ansible playbooks | ||
283 | * Collects/reports results | ||
284 | * Potentially merges change | ||
285 | |||
286 | Zuul Simulation | ||
287 | =============== | ||
288 | .. transition:: pan | ||
289 | .. container:: handout | ||
290 | |||
291 | * That was a lot of words - let's walk through it one step at a time | ||
292 | * Here we have two git repos, called nova and keystone, and their | ||
293 | current HEAD state | ||
294 | |||
295 | .. ansi:: images/zsim-00.ans | ||
296 | |||
297 | Zuul Simulation | ||
298 | =============== | ||
299 | .. transition:: cut | ||
300 | .. container:: handout | ||
301 | |||
302 | * A change is approved for Nova | ||
303 | |||
304 | .. ansi:: images/zsim-01.ans | ||
305 | |||
306 | Zuul Simulation | ||
307 | =============== | ||
308 | .. transition:: cut | ||
309 | .. container:: handout | ||
310 | |||
311 | * Zuul starts running jobs for it | ||
312 | * The tests will test the current state of nova and keystone PLUS this nova | ||
313 | change | ||
314 | |||
315 | .. ansi:: images/zsim-02.ans | ||
316 | |||
317 | Zuul Simulation | ||
318 | =============== | ||
319 | .. transition:: cut | ||
320 | .. container:: handout | ||
321 | |||
322 | * A change is approved for Keystone | ||
323 | |||
324 | .. ansi:: images/zsim-03.ans | ||
325 | |||
326 | Zuul Simulation | ||
327 | =============== | ||
328 | .. transition:: cut | ||
329 | .. container:: handout | ||
330 | |||
331 | * The tests will test the current state of nova and keystone PLUS this nova | ||
332 | change | ||
333 | |||
334 | .. ansi:: images/zsim-04.ans | ||
335 | |||
336 | Zuul Simulation | ||
337 | =============== | ||
338 | .. transition:: cut | ||
339 | .. container:: handout | ||
340 | |||
341 | * todo | ||
342 | |||
343 | .. ansi:: images/zsim-05.ans | ||
344 | |||
345 | Zuul Simulation | ||
346 | =============== | ||
347 | .. transition:: cut | ||
348 | .. container:: handout | ||
349 | |||
350 | * todo | ||
351 | |||
352 | .. ansi:: images/zsim-06.ans | ||
353 | |||
354 | Zuul Simulation | ||
355 | =============== | ||
356 | .. transition:: cut | ||
357 | .. container:: handout | ||
358 | |||
359 | * todo | ||
360 | |||
361 | .. ansi:: images/zsim-07.ans | ||
362 | |||
363 | Zuul Simulation | ||
364 | =============== | ||
365 | .. transition:: cut | ||
366 | .. container:: handout | ||
367 | |||
368 | * todo | ||
369 | |||
370 | .. ansi:: images/zsim-08.ans | ||
371 | |||
372 | Zuul Simulation | ||
373 | =============== | ||
374 | .. transition:: cut | ||
375 | .. container:: handout | ||
376 | |||
377 | * todo | ||
378 | |||
379 | .. ansi:: images/zsim-09.ans | ||
380 | |||
381 | Zuul Simulation | ||
382 | =============== | ||
383 | .. transition:: cut | ||
384 | .. container:: handout | ||
385 | |||
386 | * todo | ||
387 | |||
388 | .. ansi:: images/zsim-10.ans | ||
389 | |||
390 | Zuul Simulation | ||
391 | =============== | ||
392 | .. transition:: cut | ||
393 | .. container:: handout | ||
394 | |||
395 | * todo | ||
396 | |||
397 | .. ansi:: images/zsim-11.ans | ||
398 | |||
399 | Zuul Simulation | ||
400 | =============== | ||
401 | .. transition:: cut | ||
402 | .. container:: handout | ||
403 | |||
404 | * todo | ||
405 | |||
406 | .. ansi:: images/zsim-12.ans | ||
407 | |||
408 | Zuul Simulation | ||
409 | =============== | ||
410 | .. transition:: cut | ||
411 | .. container:: handout | ||
412 | |||
413 | * todo | ||
414 | |||
415 | .. ansi:: images/zsim-13.ans | ||
416 | |||
417 | Zuul Simulation | ||
418 | =============== | ||
419 | .. transition:: cut | ||
420 | .. container:: handout | ||
421 | |||
422 | * todo | ||
423 | |||
424 | .. ansi:: images/zsim-14.ans | ||
425 | |||
426 | Zuul Simulation | ||
427 | =============== | ||
428 | .. transition:: cut | ||
429 | .. container:: handout | ||
430 | |||
431 | * todo | ||
432 | |||
433 | .. ansi:: images/zsim-15.ans | ||
434 | |||
435 | Zuul Simulation | ||
436 | =============== | ||
437 | .. transition:: cut | ||
438 | .. container:: handout | ||
439 | |||
440 | * todo | ||
441 | |||
442 | .. ansi:: images/zsim-16.ans | ||
443 | |||
444 | Zuul Simulation | ||
445 | =============== | ||
446 | .. transition:: cut | ||
447 | .. container:: handout | ||
448 | |||
449 | * todo | ||
450 | |||
451 | .. ansi:: images/zsim-17.ans | ||
452 | |||
453 | Zuul Simulation | ||
454 | =============== | ||
455 | .. transition:: cut | ||
456 | .. container:: handout | ||
457 | |||
458 | * todo | ||
459 | |||
460 | .. ansi:: images/zsim-18.ans | ||
461 | |||
462 | Zuul Simulation | ||
463 | =============== | ||
464 | .. transition:: cut | ||
465 | .. container:: handout | ||
466 | |||
467 | * todo | ||
468 | |||
469 | .. ansi:: images/zsim-19.ans | ||
470 | |||
471 | Zuul Simulation | ||
472 | =============== | ||
473 | .. transition:: cut | ||
474 | .. container:: handout | ||
475 | |||
476 | * todo | ||
477 | |||
478 | .. ansi:: images/zsim-20.ans | ||
479 | |||
480 | Zuul Simulation | ||
481 | =============== | ||
482 | .. transition:: cut | ||
483 | .. container:: handout | ||
484 | |||
485 | * todo | ||
486 | |||
487 | .. ansi:: images/zsim-21.ans | ||
488 | |||
489 | Zuul Simulation | ||
490 | =============== | ||
491 | .. transition:: cut | ||
492 | .. container:: handout | ||
493 | |||
494 | * todo | ||
495 | |||
496 | .. ansi:: images/zsim-22.ans | ||
497 | |||
498 | Jobs | ||
499 | ==== | ||
500 | |||
501 | * Jobs run on nodes from nodepool (static or dynamic) | ||
502 | * Metadata defined in Zuul's configuration | ||
503 | * Execution content in Ansible (with live streaming!) | ||
504 | * Jobs may be defined centrally or in the repo being tested | ||
505 | * Jobs have contextual variants that simplify configuration | ||
506 | |||
507 | Shared Job Configs | ||
508 | ================== | ||
509 | |||
510 | * Job config repos are all in git | ||
511 | * Designed to support directly sharing job configurations | ||
512 | * git.zuul-ci.org/zuul-jobs repo is a 'standard library' | ||
513 | to be directly shared between zuul installations | ||
514 | |||
515 | Job | ||
516 | === | ||
517 | |||
518 | .. code:: yaml | ||
519 | |||
520 | - job: | ||
521 | name: base | ||
522 | parent: null | ||
523 | description: | | ||
524 | The base job for Zuul. | ||
525 | timeout: 1800 | ||
526 | nodeset: | ||
527 | nodes: | ||
528 | - name: primary | ||
529 | label: centos-7 | ||
530 | pre-run: playbooks/base/pre.yaml | ||
531 | post-run: | ||
532 | - playbooks/base/post-ssh.yaml | ||
533 | - playbooks/base/post-logs.yaml | ||
534 | secrets: | ||
535 | - site_logs | ||
536 | |||
537 | Simple Job | ||
538 | ========== | ||
539 | |||
540 | .. code:: yaml | ||
541 | |||
542 | - job: | ||
543 | name: tox | ||
544 | pre-run: playbooks/setup-tox.yaml | ||
545 | run: playbooks/tox.yaml | ||
546 | post-run: playbooks/fetch-tox-output.yaml | ||
547 | |||
548 | - job: | ||
549 | name: tox-py27 | ||
550 | parent: tox | ||
551 | vars: | ||
552 | tox_envlist: py27 | ||
553 | |||
554 | Playbooks | ||
555 | ========= | ||
556 | |||
557 | * Jobs run Ansible playbooks | ||
558 | * Playbooks may be defined centrally or in the repo being tested | ||
559 | * Playbooks can use roles from current or other Zuul repos | ||
560 | (or Galaxy, coming soon) | ||
561 | * Playbooks are run on the zuul-executor using bubblewrap | ||
562 | https://github.com/projectatomic/bubblewrap | ||
563 | * Playbooks are not allowed to execute content on 'localhost' | ||
564 | |||
565 | Devstack-gate / Tempest Playbook | ||
566 | ================================ | ||
567 | |||
568 | .. code:: yaml | ||
569 | |||
570 | # devstack-gate / tempest playbook | ||
571 | hosts: all | ||
572 | roles: | ||
573 | - setup-multinode-networking | ||
574 | - partition-swap | ||
575 | - configure-mirrors | ||
576 | - run-devstack | ||
577 | - run-tempest | ||
578 | |||
579 | Simple Shell Playbook | ||
580 | ===================== | ||
581 | |||
582 | .. code:: yaml | ||
583 | |||
584 | hosts: controller | ||
585 | tasks: | ||
586 | - shell: ./run_tests.sh | ||
587 | |||
588 | Test Like Production | ||
589 | ==================== | ||
590 | |||
591 | If you use Ansible for deployment, your test and deployment processes | ||
592 | and playbooks are the same | ||
593 | |||
594 | What if you don't use Ansible? | ||
595 | ============================== | ||
596 | |||
597 | OpenStack Infra Control Plane uses Puppet | ||
598 | ========================================= | ||
599 | |||
600 | .. code:: yaml | ||
601 | |||
602 | # In git.openstack.org/openstack-infra/project-config/roles/legacy-install-afs-with-puppet/tasks/main.yaml | ||
603 | - name: Install puppet | ||
604 | shell: ./install_puppet.sh | ||
605 | args: | ||
606 | chdir: "{{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/system-config" | ||
607 | environment: | ||
608 | # Skip setting up pip, our images have already done this. | ||
609 | SETUP_PIP: "false" | ||
610 | become: yes | ||
611 | |||
612 | - name: Copy manifest | ||
613 | copy: | ||
614 | src: manifest.pp | ||
615 | dest: "{{ ansible_user_dir }}/manifest.pp" | ||
616 | |||
617 | - name: Run puppet | ||
618 | puppet: | ||
619 | manifest: "{{ ansible_user_dir }}/manifest.pp" | ||
620 | become: yes | ||
621 | |||
622 | |||
623 | Cross-Project Example Problem | ||
624 | ============================= | ||
625 | |||
626 | * User reports bug in shade - auto_ip is not discovering their NAT properly | ||
627 | * Two fixes, one to detection algorithm, one to config override | ||
628 | * Config override requires adding support to os-client-config | ||
629 | * Once support is added to os-client-config, it can be consumed in shade | ||
630 | * How do we integration test this without releasing os-client-config? | ||
631 | |||
632 | Cross-Project Dependencies | ||
633 | ========================== | ||
634 | |||
635 | Testing or gating dependencies (including jobs) manually specified by | ||
636 | developers | ||
637 | |||
638 | .. container:: progressive | ||
639 | |||
640 | * shade https://review.openstack.org/513913/ | ||
641 | |||
642 | Add unittest tips jobs | ||
643 | |||
644 | Change-ID: I5b411be5c5aa43535fa89a51d6099aadd7a8ea60 | ||
645 | * os-client-config https://review.openstack.org/513915 | ||
646 | |||
647 | Add shade-tox-tips jobs | ||
648 | |||
649 | Change-ID: Ie3e9a4deca1d74b94e810e87e130706fe15fe2c9 | ||
650 | |||
651 | Depends-On: https://review.openstack.org/513913/ | ||
652 | * os-client-config https://review.openstack.org/513751/ | ||
653 | |||
654 | Added nat_source flag for networks | ||
655 | |||
656 | Change-ID: I3d8dd6d734a1013d2d4a43e11c3538c3a345820b | ||
657 | |||
658 | * shade https://review.openstack.org/#/c/513914 | ||
659 | |||
660 | Add support for configured NAT source variable | ||
661 | |||
662 | Change-Id: I4b50c2323a487b5ce90f9d38a48be249cfb739c5 | ||
663 | |||
664 | Depends-On: https://review.openstack.org/513914 | ||
665 | |||
666 | shade: Add unittest tips jobs | ||
667 | ============================= | ||
668 | |||
669 | * In git.openstack.org/openstack-infra/shade/.zuul.yaml: | ||
670 | |||
671 | .. code:: yaml | ||
672 | |||
673 | - job: | ||
674 | name: shade-tox-py27-tips | ||
675 | parent: openstack-tox-py27 | ||
676 | description: | | ||
677 | Run tox python 27 unittests against master of important libs | ||
678 | required-projects: | ||
679 | - openstack-infra/shade | ||
680 | - openstack/keystoneauth | ||
681 | - openstack/os-client-config | ||
682 | |||
683 | - job: | ||
684 | name: shade-tox-py35-tips | ||
685 | parent: openstack-tox-py35 | ||
686 | description: | | ||
687 | Run tox python 35 unittests against master of important libs | ||
688 | required-projects: | ||
689 | - openstack-infra/shade | ||
690 | - openstack/keystoneauth | ||
691 | - openstack/os-client-config | ||
692 | |||
693 | shade: Add unittest tips project-template | ||
694 | ========================================= | ||
695 | |||
696 | * In git.openstack.org/openstack-infra/shade/.zuul.yaml: | ||
697 | |||
698 | .. code:: yaml | ||
699 | |||
700 | - project-template: | ||
701 | name: shade-tox-tips | ||
702 | check: | ||
703 | jobs: | ||
704 | - shade-tox-py27-tips | ||
705 | - shade-tox-py35-tips | ||
706 | gate: | ||
707 | jobs: | ||
708 | - shade-tox-py27-tips | ||
709 | - shade-tox-py35-tips | ||
710 | |||
711 | shade: Add unittest tips project-template to project | ||
712 | ==================================================== | ||
713 | |||
714 | * In git.openstack.org/openstack-infra/shade/.zuul.yaml: | ||
715 | |||
716 | .. code:: yaml | ||
717 | |||
718 | - project: | ||
719 | templates: | ||
720 | - publish-to-pypi | ||
721 | - publish-openstack-sphinx-docs | ||
722 | - shade-tox-tips | ||
723 | |||
724 | os-client-config: Add shade-tox-tips jobs | ||
725 | ========================================= | ||
726 | |||
727 | * In git.openstack.org/openstack/os-client-config/.zuul.yaml: | ||
728 | |||
729 | .. code:: yaml | ||
730 | |||
731 | - project: | ||
732 | templates: | ||
733 | - shade-tox-tips | ||
734 | |||
735 | os-client-config: Add nat_source flag for networks | ||
736 | ================================================== | ||
737 | |||
738 | :: | ||
739 | |||
740 | diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py | ||
741 | index 2e97629..d1a6983 100644 | ||
742 | --- a/os_client_config/cloud_config.py | ||
743 | +++ b/os_client_config/cloud_config.py | ||
744 | @@ -581,3 +581,10 @@ class CloudConfig(object): | ||
745 | if net['nat_destination']: | ||
746 | return net['name'] | ||
747 | return None | ||
748 | + | ||
749 | + def get_nat_source(self): | ||
750 | + """Get network used for NAT source.""" | ||
751 | + for net in self.config['networks']: | ||
752 | + if net.get('nat_source'): | ||
753 | + return net['name'] | ||
754 | + return None | ||
755 | |||
756 | shade: Add support for configured NAT source variable | ||
757 | ===================================================== | ||
758 | |||
759 | :: | ||
760 | |||
761 | Zuul 10-21 13:57 | ||
762 | Patch Set 5: Verified-1 | ||
763 | Build failed. | ||
764 | openstack-tox-pep8 SUCCESS in 2m 29s | ||
765 | openstack-tox-py27 FAILURE in 2m 34s | ||
766 | build-openstack-releasenotes SUCCESS in 2m 47s | ||
767 | openstack-tox-py35 FAILURE in 2m 41s | ||
768 | openstack-tox-cover POST_FAILURE in 3m 52s (non-voting) | ||
769 | build-openstack-sphinx-docs SUCCESS in 2m 57s | ||
770 | shade-tox-py27-tips SUCCESS in 3m 18s | ||
771 | shade-tox-py35-tips SUCCESS in 2m 28s | ||
772 | |||
773 | OpenStack Github Support for Cross Community Testing | ||
774 | ==================================================== | ||
775 | |||
776 | * OpenStack does not use Github, but other people do | ||
777 | * Github App "OpenStack Zuul" | ||
778 | * App added to github project by project admin | ||
779 | * Project aded to OpenStack's main.yaml | ||
780 | * Test interactions between OpenStack and important adjacent communities | ||
781 | * https://github.com/ansible/ansible/pull/20974 | ||
782 | |||
783 | Cross Source Dependencies | ||
784 | ========================= | ||
785 | |||
786 | .. container:: progressive | ||
787 | |||
788 | * shade https://review.openstack.org/539563 | ||
789 | |||
790 | Shift voting flag and test_matrix_branch for ansible-devel job | ||
791 | |||
792 | Change-ID: Ic9d3983de641dbe618c65b2cbf2dcfa3686575df | ||
793 | |||
794 | * ansible https://github.com/ansible/ansible/pull/34925 | ||
795 | |||
796 | continue fact gathering even without dmidecode | ||
797 | |||
798 | * ansible https://github.com/ansible/ansible/pull/20974 | ||
799 | |||
800 | Make a generalized OpenStack cloud constructor | ||
801 | |||
802 | Depends-On: https://review.openstack.org/539563 | ||
803 | Depends-On: https://github.com/ansible/ansible/pull/34925 | ||
804 | |||
805 | Nodesets for Multi-node Jobs | ||
806 | ============================ | ||
807 | |||
808 | .. code:: yaml | ||
809 | |||
810 | - nodeset: | ||
811 | name: ceph-cluster | ||
812 | nodes: | ||
813 | - name: controller | ||
814 | label: centos-7 | ||
815 | - name: compute1 | ||
816 | label: fedora-26 | ||
817 | - name: compute2 | ||
818 | label: fedora-26 | ||
819 | groups: | ||
820 | - name: ceph-osd | ||
821 | nodes: | ||
822 | - controller | ||
823 | - name: ceph-monitor | ||
824 | nodes: | ||
825 | - controller | ||
826 | - compute1 | ||
827 | - compute2 | ||
828 | |||
829 | Multi-node Job | ||
830 | ============== | ||
831 | |||
832 | * nodesets are provided to Ansible for jobs in inventory | ||
833 | |||
834 | .. code:: yaml | ||
835 | |||
836 | - job: | ||
837 | name: ceph-multinode | ||
838 | nodeset: ceph-cluster | ||
839 | run: playbooks/install-ceph.yaml | ||
840 | |||
841 | Multi-node Ceph Job Content | ||
842 | =========================== | ||
843 | |||
844 | .. code:: yaml | ||
845 | |||
846 | - hosts: all | ||
847 | roles: | ||
848 | - install-ceph | ||
849 | - hosts: ceph-osd | ||
850 | roles: | ||
851 | - start-ceph-osd | ||
852 | - hosts: ceph-monitor | ||
853 | roles: | ||
854 | - start-ceph-monitor | ||
855 | - hosts: all | ||
856 | roles: | ||
857 | - do-something-interesting | ||
858 | |||
859 | Projects | ||
860 | ======== | ||
861 | |||
862 | * Projects are git repositories | ||
863 | * Specify a set of jobs for each pipeline | ||
864 | * golang git repo naming as been adopted: | ||
865 | |||
866 | :: | ||
867 | |||
868 | zuul@ubuntu-xenial:~$ find /home/zuul/src -mindepth 3 -maxdepth 3 -type d | ||
869 | /home/zuul/src/git.openstack.org/openstack-infra/shade | ||
870 | /home/zuul/src/git.openstack.org/openstack/keystoneauth | ||
871 | /home/zuul/src/git.openstack.org/openstack/os-client-config | ||
872 | /home/zuul/src/github.com/ansible/ansible | ||
873 | |||
874 | Project with Job Dependencies | ||
875 | ============================= | ||
876 | |||
877 | .. code:: yaml | ||
878 | |||
879 | # In git.openstack.org/openstack-infra/project-config: | ||
880 | - project: | ||
881 | name: openstack/nova | ||
882 | release: | ||
883 | jobs: | ||
884 | - build-artifacts | ||
885 | - upload-tarball: | ||
886 | dependencies: build-artifacts | ||
887 | - upload-pypi: | ||
888 | dependencies: build-artifacts | ||
889 | - notify-mirror: | ||
890 | dependencies: | ||
891 | - upload-tarball | ||
892 | - upload-pypi | ||
893 | |||
894 | Secrets | ||
895 | ======= | ||
896 | |||
897 | * Inspired by Kubernetes Secrets API | ||
898 | * Projects can add named encrypted secrets to their .zuul.yaml file | ||
899 | * Jobs can request to use secrets by name | ||
900 | * Jobs using secrets are not reconfigured speculatively | ||
901 | * Secrets can only be used by the same project they are defined in | ||
902 | * Public key per project: | ||
903 | ``{{ zuul_url }}/tenant/{{ tenant }}/key/{{ project }}.pub`` | ||
904 | |||
905 | Secret Example (note, no admins had to enable this) | ||
906 | =================================================== | ||
907 | |||
908 | .. code:: yaml | ||
909 | |||
910 | # In git.openstack.org/openstack/loci/.zuul.yaml: | ||
911 | - secret: | ||
912 | name: loci_docker_login | ||
913 | data: | ||
914 | user: !encrypted/pkcs1-oaep | ||
915 | - r8Nbpq5olmfLF035BZ/CUoFLIdhvBi/49KuochOAHbvns+xMiho3C7MEFzYDqJX3IhHde | ||
916 | BICYOgK7qnyINOIZL2e7pl75rEdHQwJjSFUMkpdY6wEP7f9hpolj9xVp0ifHUVQqPHMRn | ||
917 | zoPFd8MEAHxH5GLmc2SWJ98E/QUqGltxBi1YRSZoCcNtq3tHFK5Y+xQlLhIseJ2HkpDs6 | ||
918 | YXOGP9Qt4Va6sdyBcA90H+apSAcYA3Duu962ySZQAsYNui/3NQq3gLA+OZeyTJtcrh4hj | ||
919 | Rb5dBnDWfSrMpxdNkbPXXgbQaxO3T0L4jbaOF8VKEsiI9olBrOeV2M9ddYJjSsHGj4XR8 | ||
920 | 4vwS0+doB7np93fujiDuHVgdG8R40NW2GznyKRlRtzAORla7Mzw1Y1MokcUyY6p1LlLLl | ||
921 | wUuWYCCEuRciOPhZXQ2u42qju/zrK2/dPnO8HfUINSrN0WbNq14ZwPpbj0ro02oGPbtwu | ||
922 | OTw1z+N0Nc+GuLWlwYJGYM/z0UnvDR3WEBc2kXbVev9w4n0cB3RyphML2PDZZWbw8tjnX | ||
923 | h1VsAOJ0Qo4qq1K/ft95ypd+vtjkfepEgHEBmJNwutJa9IHAkGfrkO9VkpUTPpfffnPwz | ||
924 | d0/zaaadNl6MLQUSutRwY23YIIbv+fmukxw2vnJmvn6abkBlMya7KgtifwNA8c= | ||
925 | password: !encrypted/pkcs1-oaep | ||
926 | - gUEX4eY3JAk/Xt7Evmf/hF7xr6HpNRXTibZjrKTbmI4QYHlzEBrBbHey27Pt/eYvKKeKw | ||
927 | hk8MDQ4rNX7ZK1v+CKTilUfOf4AkKYbe6JFDd4z+zIZ2PAA7ZedO5FY/OnqrG7nhLvQHE | ||
928 | 5nQrYwmxRp4O8eU5qG1dSrM9X+bzri8UnsI7URjqmEsIvlUqtybQKB9qQXT4d6mOeaKGE | ||
929 | 5h6Ydkb9Zdi4Qh+GpCGDYwHZKu1mBgVK5M1G6NFMy1DYz+4NJNkTRe9J+0TmWhQ/KZSqo | ||
930 | 4ck0x7Tb0Nr7hQzV8SxlwkaCTLDzvbiqmsJPLmzXY2jry6QsaRCpthS01vnj47itoZ/7p | ||
931 | taH9CoJ0Gl7AkaxsrDSVjWSjatTQpsy1ub2fuzWHH4ASJFCiu83Lb2xwYts++r8ZSn+mA | ||
932 | hbEs0GzPI6dIWg0u7aUsRWMOB4A+6t2IOJibVYwmwkG8TjHRXxVCLH5sY+i3MR+NicR9T | ||
933 | IZFdY/AyH6vt5uHLQDU35+5n91pUG3F2lyiY5aeMOvBL05p27GTMuixR5ZoHcvSoHHtCq | ||
934 | 7Wnk21iHqmv/UnEzqUfXZOque9YP386RBWkshrHd0x3OHUfBK/WrpivxvIGBzGwMr2qAj | ||
935 | /AhJsfDXKBBbhGOGk1u5oBLjeC4SRnAcIVh1+RWzR4/cAhOuy2EcbzxaGb6VTM= | ||
936 | |||
937 | Secret Example | ||
938 | ============== | ||
939 | |||
940 | .. code:: yaml | ||
941 | |||
942 | # In git.openstack.org/openstack/loci/.zuul.yaml: | ||
943 | - job: | ||
944 | name: publish-loci-cinder | ||
945 | parent: loci-cinder | ||
946 | post-run: playbooks/push | ||
947 | secrets: | ||
948 | - loci_docker_login | ||
949 | |||
950 | # In git.openstack.org/openstack/loci/playbooks/push.yaml: | ||
951 | - hosts: all | ||
952 | tasks: | ||
953 | - include_vars: vars.yaml | ||
954 | |||
955 | - name: Push project to DockerHub | ||
956 | block: | ||
957 | - command: docker login -u {{ loci_docker_login.user }} -p {{ loci_docker_login.password }} | ||
958 | no_log: True | ||
959 | - command: docker push openstackloci/{{ project }}:{{ branch }}-{{ item.name }} | ||
960 | with_items: "{{ distros }}" | ||
961 | |||
962 | |||
963 | What's Next? | ||
964 | ============ | ||
965 | |||
966 | * shared job doc generation | ||
967 | * node providers | ||
968 | * kuberenetes | ||
969 | * OCI/docker | ||
970 | * Mac Stadium (for our Ansible friends) | ||
971 | * ec2 | ||
972 | * native container/kubernetes job execution | ||
973 | |||
974 | Important Links | ||
975 | =============== | ||
976 | |||
977 | * https://zuul-ci.org/ | ||
978 | * https://git.openstack.org/cgit/openstack-infra/zuul | ||
979 | * https://docs.openstack.org/infra/zuul | ||
980 | * https://docs.openstack.org/infra/manual/zuulv3.html | ||
981 | * https://docs.openstack.org/infra/zuul-jobs/ | ||
982 | * https://docs.openstack.org/infra/openstack-zuul-jobs/ | ||
983 | * https://storyboard.openstack.org/#!/project/679 | ||
984 | * https://storyboard.openstack.org/#!/board/41 | ||
985 | * freenode:#zuul | ||
986 | |||
987 | Questions | ||
988 | ========= | ||
989 | |||
990 | .. ansi:: images/questions.ans | ||
991 | |||
992 | Presentty | ||
993 | ========= | ||
994 | .. hidetitle:: | ||
995 | .. transition:: pan | ||
996 | .. figlet:: Presentty | ||
997 | |||
998 | * Console presentations written in reStructuredText | ||
999 | * Cross-fade, pan, tilt, cut transitions | ||
1000 | * Figlet, cowsay! | ||
1001 | * https://pypi.python.org/pypi/presentty | ||