1. commit 38639fed9957f40c71ab1cd0ab767c33c79ca6b6
  2. Author: Loic Dachary <[email protected]>
  3. Date: Thu Nov 5 07:54:35 2015 +0100
  5. jenkins: no longer use jenkins
  7. Signed-off-by: Loic Dachary <[email protected]>
  9. diff --git a/ChangeLog b/ChangeLog
  10. index 3efedea..712739e 100644
  11. --- a/ChangeLog
  12. +++ b/ChangeLog
  13. @@ -1,4 +1,4 @@
  14. -commit c9ed7e52758997a38724d549d764fa6d9b3b3193
  15. +commit 36232fe550a9a07bbfcfdcdec6fd651186ba2635
  16. Author: Loic Dachary <[email protected]>
  17. Date: Tue Apr 28 12:04:04 2015 +0200
  19. diff --git a/ceph_workbench/jenkins.py b/ceph_workbench/jenkins.py
  20. deleted file mode 100644
  21. index e01df85..0000000
  22. --- a/ceph_workbench/jenkins.py
  23. +++ /dev/null
  24. @@ -1,575 +0,0 @@
  25. -# -*- mode: python; coding: utf-8 -*-
  26. -#
  27. -# Copyright (C) 2015 <[email protected]>
  28. -#
  29. -# Author: Loic Dachary <[email protected]>
  30. -#
  31. -# This program is free software: you can redistribute it and/or modify
  32. -# it under the terms of the GNU Affero General Public License as
  33. -# published by the Free Software Foundation, either version 3 of the
  34. -# License, or (at your option) any later version.
  35. -#
  36. -# This program is distributed in the hope that it will be useful,
  37. -# but WITHOUT ANY WARRANTY; without even the implied warranty of
  38. -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  39. -# GNU Affero General Public License for more details.
  40. -#
  41. -# You should have received a copy of the GNU Affero General Public License
  42. -# along with this program. If not, see `<http://www.gnu.org/licenses/>`.
  43. -#
  44. -import argparse
  45. -import grp
  46. -import json
  47. -import logging
  48. -import os
  49. -import re
  50. -import requests
  51. -import shutil
  52. -import subprocess
  53. -import time
  54. -
  55. -from ceph_workbench import util
  56. -
  57. -
  58. -class NoCredentials(Exception):
  59. - pass
  60. -
  61. -cred_template = """
  62. -{
  63. - "domainCredentials": {
  64. - "domain": {
  65. - "name": "",
  66. - "description": ""
  67. - },
  68. - "credentials": [
  69. - {
  70. - "scope": "GLOBAL",
  71. - "id": "",
  72. - "username": "{USERNAME}",
  73. - "description": "{DESCRIPTION}",
  74. - "privateKeySource": {
  75. - "value": "1",
  76. - "privateKeyFile": "{KEY_FILE}",
  77. - "stapler-class": "com.cloudbees.jenkins.plugins\
  78. -.sshcredentials.impl.BasicSSHUserPrivateKey$FileOnMasterPrivateKeySource"
  79. - },
  80. - "passphrase": "",
  81. - "stapler-class": "com.cloudbees.jenkins.plugins\
  82. -.sshcredentials.impl.BasicSSHUserPrivateKey",
  83. - "kind": "com.cloudbees.jenkins.plugins\
  84. -.sshcredentials.impl.BasicSSHUserPrivateKey"
  85. - }
  86. - ]
  87. - }
  88. -}
  89. -"""
  90. -
  91. -
  92. -class Jenkins:
  93. -
  94. - def __init__(self, args):
  95. - self.args = args
  96. - self.args.port_range = 3000
  97. - self.args.jenkins_slave = 'ceph-jenkins-slave'
  98. - self.args.jenkins_master = 'ceph-jenkins-master'
  99. - self.args.libguestfs = 'libguestfs'
  100. - self.args.jar_dir = '/tmp'
  101. -
  102. - def run(self):
  103. - if self.args.publish_build_to_pull_request:
  104. - return self.publish_build_to_pull_request()
  105. - elif self.args.jenkins_make_check_bot:
  106. - return subprocess.call([self.args.libdir +
  107. - '/docker-make-check-bot'])
  108. - elif self.args.jenkins_docker_run_slave:
  109. - return self.run_slave('docker', self.args.jenkins_docker_run_slave)
  110. - elif self.args.jenkins_qemu_prepare:
  111. - return self.prepare_qemu_img(self.args.jenkins_qemu_image)
  112. - elif self.args.jenkins_qemu_run_slave:
  113. - return self.run_slave('qemu', self.args.jenkins_qemu_run_slave)
  114. - elif self.args.jenkins_docker_destroy_slave:
  115. - return self.docker_destroy_slave(
  116. - self.args.jenkins_docker_destroy_slave)
  117. - elif self.args.jenkins_run_master:
  118. - return self.run_master()
  119. -
  120. - @staticmethod
  121. - def get_parser():
  122. - parser = argparse.ArgumentParser(
  123. - description="Jenkins",
  124. - conflict_handler='resolve',
  125. - )
  126. - parser.add_argument('--jenkins-run-master',
  127. - action='store_true',
  128. - help='run the jenkins master')
  129. - parser.add_argument('--jenkins-docker-run-slave',
  130. - help=('run the jenkins slave in docker and '
  131. - 'register it to the jenkins master'))
  132. - parser.add_argument('--jenkins-docker-destroy-slave',
  133. - help='destroy the jenkins docker slave')
  134. - parser.add_argument('--jenkins-qemu-run-slave',
  135. - help=('run the jenkins slave in qemu and '
  136. - 'register it to the jenkins master'))
  137. - parser.add_argument('--jenkins-qemu-prepare',
  138. - action='store_true',
  139. - help=('create a directory with '
  140. - 'a ready to use image'))
  141. - parser.add_argument('--jenkins-qemu-image',
  142. - help='for instance ubuntu-14.04-i386')
  143. - parser.add_argument('--jenkins-qemu-ram',
  144. - default='512',
  145. - help='RAM for qemu, in GB')
  146. - parser.add_argument('--jenkins-qemu-ssh-port',
  147. - help='ssh port to connect to the qemu instance')
  148. - parser.add_argument('--jenkins-make-check-bot',
  149. - help='run the jenkins-make-check-bot helper')
  150. - parser.add_argument('--publish-build-to-pull-request',
  151. - help='add a comment to pull request XXX')
  152. - parser.add_argument('--jenkins-git-branch',
  153. - help=('only for --publish-build-to-pull-request, '
  154. - 'force the GIT_BRANCH value'))
  155. - parser.add_argument('--jenkins-host',
  156. - help='hostname of the jenkins master')
  157. - parser.add_argument('--jenkins-port',
  158. - default='80',
  159. - help='port of the jenkins master')
  160. - parser.add_argument('--jenkins-ssh-host',
  161. - help='jenkins host name for ssh protocol')
  162. - parser.add_argument('--jenkins-ssh-port',
  163. - default='22',
  164. - help='jenkins port for ssh protocol')
  165. - parser.add_argument('--jenkins-ssh-key',
  166. - help='jenkins ssh private key')
  167. - parser.add_argument('--jenkins-user',
  168. - help='jenkins user name for API authentication')
  169. - parser.add_argument('--jenkins-password',
  170. - help='jenkins password for API authentication')
  171. - parser.add_argument('--github-repo',
  172. - help='GitHub repo (for instance ceph/ceph)')
  173. - parser.add_argument('--github-token',
  174. - help='GitHub authentication token')
  175. - parser.add_argument('--libdir',
  176. - help='directory containing helpers programs')
  177. - parser.add_argument('--datadir',
  178. - help='directory for persistent data')
  179. - parser.add_argument('--jenkins-home',
  180. - help='jenkins home directory, on the docker host',
  181. - default=os.environ['HOME'])
  182. - return parser
  183. -
  184. - @staticmethod
  185. - def factory(argv):
  186. - return Jenkins(Jenkins.get_parser().parse_args(argv))
  187. -
  188. - def get_url(self):
  189. - return ('http://' + self.args.jenkins_host +
  190. - ':' + self.args.jenkins_port)
  191. -
  192. - def publish_build_to_pull_request(self):
  193. - r = requests.get(self.get_url() + '/' +
  194. - self.args.publish_build_to_pull_request + '/api/json')
  195. - r.raise_for_status()
  196. - report = r.json()
  197. - logging.debug("publish_build_to_pull_request: downstream " +
  198. - str(report))
  199. - for action in report['actions']:
  200. - if "causes" in action:
  201. - cause = action['causes'][0]
  202. - upstream = (cause['upstreamUrl'] + '/' +
  203. - str(cause['upstreamBuild']))
  204. - r = requests.get(self.get_url() + '/' + upstream + '/api/json')
  205. - r.raise_for_status()
  206. - report = r.json()
  207. - logging.debug("publish_build_to_pull_request: upstream " + str(report))
  208. - if self.args.jenkins_git_branch:
  209. - branch_name = self.args.jenkins_git_branch
  210. - else:
  211. - for action in report['actions']:
  212. - if "lastBuiltRevision" in action:
  213. - branch = action["lastBuiltRevision"]["branch"][0]
  214. - branch_name = branch['name']
  215. - runs = {}
  216. - for run in report['runs']:
  217. - if run['number'] == report['number']:
  218. - r = requests.get(run['url'] + 'api/json')
  219. - r.raise_for_status()
  220. - runs[run['url']] = r.json()['result']
  221. - body = report['result'] + ": " + report['url'] + "\n"
  222. - for url in sorted(runs):
  223. - body += "* " + runs[url] + " " + url + "/console\n"
  224. - logging.debug("publish_build_to_pull_request: " + body)
  225. - if 'pull/' in branch_name:
  226. - (pr,) = re.findall('\d+$', branch_name)
  227. - payload = {'body': body}
  228. - r = requests.post('https://api.github.com/repos/' +
  229. - self.args.github_repo + '/issues/' +
  230. - pr + '/comments?access_token=' +
  231. - self.args.github_token,
  232. - data=json.dumps(payload))
  233. - r.raise_for_status()
  234. - return r.json()['body']
  235. - else:
  236. - return body
  237. -
  238. - def get_cli_jar(self):
  239. - return self.args.jar_dir + '/jenkins-cli.jar'
  240. -
  241. - def run_cli(self, *args, **kwargs):
  242. - jar = self.get_cli_jar()
  243. - if not os.path.exists(jar):
  244. - r = requests.get(self.get_url() +
  245. - '/jnlpJars/jenkins-cli.jar')
  246. - with open(jar, 'wb') as fd:
  247. - for chunk in r.iter_content(4096):
  248. - fd.write(chunk)
  249. - cmd = ['java', '-jar', jar, '-s', self.get_url()] + list(args)
  250. - s = subprocess.Popen(cmd,
  251. - stdin=subprocess.PIPE,
  252. - stdout=subprocess.PIPE,
  253. - stderr=subprocess.PIPE)
  254. - if 'payload' in kwargs:
  255. - payload = kwargs['payload'].encode('utf-8')
  256. - else:
  257. - payload = None
  258. - (stdoutdata, stderrdata) = s.communicate(input=payload)
  259. - stdoutdata = stdoutdata.decode('utf-8')
  260. - stderrdata = stderrdata.decode('utf-8')
  261. - logging.debug(" ".join(cmd) + ":" +
  262. - " stdout " + stdoutdata +
  263. - " stderr " + stderrdata)
  264. - return stdoutdata + stderrdata
  265. -
  266. - def master_port(self, name):
  267. - (number,) = re.findall('(\d+)$', name)
  268. - return str(self.args.port_range + int(number))
  269. -
  270. - def slave_port(self, name):
  271. - (number,) = re.findall('(\d+)$', name)
  272. - return str(4000 + int(number))
  273. -
  274. - def find_node(self, name):
  275. - r = requests.get(self.get_url() + '/computer/api/json')
  276. - r.raise_for_status()
  277. - return name in map(lambda x: x['displayName'], r.json()['computer'])
  278. -
  279. - def destroy_node(self, name):
  280. - if self.find_node(name):
  281. - self.run_cli("delete-node", name)
  282. -
  283. - def next_node_name(self):
  284. - r = requests.get(self.get_url() + '/computer/api/json')
  285. - r.raise_for_status()
  286. - nodes = r.json()['computer']
  287. - highest = 0
  288. - for node in nodes:
  289. - name_with_number = re.findall('(\d+)$', node['displayName'])
  290. - if name_with_number:
  291. - highest = max(highest, int(name_with_number[0]))
  292. - highest += 1
  293. - return "slave%03d" % highest
  294. -
  295. - def create_node(self, name):
  296. - # assume the master is running in docker and slaves are accessed
  297. - # via a ssh tunel open on the docker host
  298. - credentials = self.slave_credential_id()
  299. - port = self.master_port(name)
  300. - payload = """
  301. -<?xml version="1.0" encoding="UTF-8"?>
  302. -<slave>
  303. - <name>{name}</name>
  304. - <description></description>
  305. - <remoteFS>/home/jenkins</remoteFS>
  306. - <numExecutors>1</numExecutors>
  307. - <mode>NORMAL</mode>
  308. - <retentionStrategy class="hudson.slaves.RetentionStrategy$Always"/>
  309. - <launcher class="hudson.plugins.sshslaves.SSHLauncher"
  310. - plugin="[email protected]">
  311. - <host>localhost</host>
  312. - <port>{port}</port>
  313. - <credentialsId>{credentials}</credentialsId>
  314. - <maxNumRetries>0</maxNumRetries>
  315. - <retryWaitTime>0</retryWaitTime>
  316. - </launcher>
  317. - <label>docker ceph-workbench</label>
  318. - <nodeProperties/>
  319. - <userId>admin</userId>
  320. -</slave>
  321. -""".format(port=port,
  322. - name=name,
  323. - credentials=credentials)
  324. - logging.debug('create_node: ' + name + ' ' + payload)
  325. - self.run_cli('create-node', name, payload=payload)
  326. - r = requests.get(self.get_url() +
  327. - '/computer/' + name + '/api/json')
  328. - r.raise_for_status()
  329. - return r.json()
  330. -
  331. - def run_slave(self, kind, name):
  332. - if name == 'new':
  333. - name = self.next_node_name()
  334. - self.create_node(name)
  335. - if kind == 'docker':
  336. - return self.docker_run_slave(name)
  337. - elif kind == 'qemu':
  338. - return self.qemu_run_slave(name)
  339. - else:
  340. - raise Exception(kind + ' is not docker nor qemu')
  341. -
  342. - def slave_credential_id(self):
  343. - auth = (self.args.jenkins_user, self.args.jenkins_password)
  344. - r = requests.get(self.get_url() +
  345. - '/credential-store/domain/_/api/json',
  346. - auth=auth)
  347. - r.raise_for_status()
  348. - slave_description = 'for jenkins slaves'
  349. - credentials = {}
  350. - for credential in list(r.json()['credentials']):
  351. - c = requests.get(self.get_url() +
  352. - ('/credential-store/domain/_/credential/' +
  353. - credential + '/api/json'),
  354. - auth=auth)
  355. - c.raise_for_status()
  356. - credentials[c.json()['description']] = credential
  357. -
  358. - if slave_description in credentials:
  359. - return credentials[slave_description]
  360. - else:
  361. - raise NoCredentials('no "' + slave_description +
  362. - '" credential in ' +
  363. - str(credentials))
  364. -
  365. - def fetch_ssh_key(self, directory):
  366. - if not os.path.exists(directory):
  367. - os.makedirs(directory, 0o755)
  368. - dotssh = directory + '/.ssh'
  369. - if not os.path.exists(dotssh):
  370. - os.mkdir(dotssh, 0o700)
  371. - if self.args.jenkins_ssh_key:
  372. - key = '-i ' + self.args.jenkins_ssh_key + ' '
  373. - else:
  374. - key = ''
  375. - if self.args.jenkins_ssh_port:
  376. - port = '-P ' + self.args.jenkins_ssh_port + ' '
  377. - else:
  378. - port = ''
  379. - known_hosts = self.ignore_known_hosts()
  380. - util.sh('scp ' + known_hosts + key + port +
  381. - self.args.jenkins_ssh_host +
  382. - ':/var/jenkins_home/.ssh/id_rsa* ' +
  383. - dotssh)
  384. - shutil.copyfile(dotssh + '/id_rsa.pub',
  385. - dotssh + '/authorized_keys')
  386. - os.chmod(dotssh + '/authorized_keys', 0o600)
  387. -
  388. - @staticmethod
  389. - def ignore_known_hosts():
  390. - return ('-o UserKnownHostsFile=/dev/null '
  391. - '-o StrictHostKeyChecking=no ')
  392. -
  393. - def tunnel(self, port):
  394. - script = ("ssh {known_hosts} " +
  395. - " -p {jenkins_port} -f -N -R {port}:localhost:22 " +
  396. - " {jenkins_host}")
  397. - return script.format(known_hosts=self.ignore_known_hosts(),
  398. - port=port,
  399. - jenkins_port=self.args.jenkins_ssh_port,
  400. - jenkins_host=self.args.jenkins_ssh_host)
  401. -
  402. - def qemu_get_img(self):
  403. - image = self.args.jenkins_qemu_image
  404. - image_dir = self.args.datadir + "/" + image
  405. - if (os.path.exists(image_dir) and
  406. - os.path.exists(image_dir + '/.ssh')):
  407. - return image_dir
  408. - elif self.docker_p():
  409. - return self.prepare_qemu_img(image)
  410. - else:
  411. - host = self.args.jenkins_ssh_host
  412. - util.sh("rsync -avz " + host + ":" + image + "/ " +
  413. - image_dir + "/")
  414. - return image_dir
  415. -
  416. - def qemu_ssh(self, name, command):
  417. - slave_port = self.slave_port(name)
  418. - known_hosts = self.ignore_known_hosts()
  419. - image = self.args.jenkins_qemu_image
  420. - key = self.args.datadir + '/' + image + '/.ssh/id_rsa'
  421. - return util.sh("ssh -p " + slave_port + " " + known_hosts + " " +
  422. - " -i " + key + " jenkins@localhost " + command)
  423. -
  424. - def qemu_destroy_slave(self, name):
  425. - self.qemu_ssh(name, 'sudo halt -p')
  426. - return self.destroy_node(name)
  427. -
  428. - def qemu_run_slave(self, name):
  429. - image = self.args.jenkins_qemu_image
  430. - image_dir = self.qemu_get_img()
  431. - ram = self.args.jenkins_qemu_ram
  432. - if not os.path.exists(image_dir + '/' + image + '.img'):
  433. - shutil.copyfile(image_dir + '/' + image + '.orig',
  434. - image_dir + '/' + image + '.img')
  435. - master_port = self.master_port(name)
  436. - slave_port = self.slave_port(name)
  437. - util.sh(('cd {image_dir} ; ' +
  438. - 'kvm -name {name} ' +
  439. - '-m {ram} -smp {cpus},sockets=1 ' +
  440. - '-drive file={image}.img,if=virtio ' +
  441. - '-net nic -net user,hostfwd=tcp::{slave_port}-:22 ' +
  442. - '-kernel vmlinuz-* -initrd initrd.* ' +
  443. - '-append "root=/dev/vda1 ro ds=nocloud-net ' +
  444. - ' brd.rd_size={rd_size} ' +
  445. - ' master_port={master_port}" ' +
  446. - '-display none').format(ram=ram,
  447. - slave_port=slave_port,
  448. - master_port=master_port,
  449. - name=name,
  450. - cpus=str(8),
  451. - rd_size=str(16 * 1024 * 1024),
  452. - image=image,
  453. - image_dir=image_dir))
  454. -
  455. - def prepare_qemu_img(self, image):
  456. - self.docker_build_libguestfs()
  457. - libdir = os.path.abspath(self.args.libdir)
  458. - datadir = os.path.abspath(self.args.datadir)
  459. - image_dir = datadir + '/' + image
  460. - self.fetch_ssh_key(image_dir)
  461. - script = """\
  462. -#!/bin/sh -e
  463. -mkfs -t ext4 /dev/ram1
  464. -mkdir -p /home/jenkins/workspace
  465. -mount /dev/ram1 /home/jenkins/workspace
  466. -chown jenkins /home/jenkins/workspace
  467. -port=$(perl -ne 'print if(s/.*master_port=([0-9]*).*/$1/)' < /proc/cmdline)
  468. -if test "$port" ; then
  469. - su - -c "{tunnel}" jenkins &
  470. -fi
  471. - """.format(tunnel=self.tunnel('$port'))
  472. - open(image_dir + '/rc.local', 'w').write(script)
  473. - util.sh('docker run --rm ' +
  474. - ' -v ' + datadir + ':' + datadir +
  475. - ' -v ' + libdir + ':' + libdir +
  476. - ' -w ' + datadir +
  477. - ' ' + self.args.libguestfs + ' ' + libdir + '/' +
  478. - 'create-qemu-img ' + image + ' ' + str(os.getuid()))
  479. - return True
  480. -
  481. - def docker_p(self):
  482. - try:
  483. - self.docker_version()
  484. - return True
  485. - except subprocess.CalledProcessError:
  486. - return False
  487. -
  488. - def docker_version(self):
  489. - return util.sh("docker --version | "
  490. - " perl -pe 's/.*version (.*),.*/\\1/'")
  491. -
  492. - def docker_build_slave(self):
  493. - if self.args.jenkins_slave not in util.sh('docker images'):
  494. - util.sh('docker build -t ' + self.args.jenkins_slave + " " +
  495. - self.args.libdir + '/jenkins-slave-docker')
  496. - return True
  497. - return False
  498. -
  499. - def docker_build_libguestfs(self):
  500. - if self.args.libguestfs not in util.sh('docker images'):
  501. - util.sh('docker build -t ' + self.args.libguestfs + " " +
  502. - self.args.libdir + '/libguestfs-docker')
  503. - return True
  504. - return False
  505. -
  506. - def docker_destroy_slave(self, name):
  507. - util.sh('docker stop ' + name + ' || true')
  508. - util.sh('docker rm ' + name + ' || true')
  509. - return self.destroy_node(name)
  510. -
  511. - def docker_run_slave(self, name):
  512. - self.docker_build_slave()
  513. - slave_port = self.slave_port(name)
  514. - docker_host_version = util.sh("docker --version | "
  515. - " perl -pe 's/.*version (.*),.*/\\1/'")
  516. - master_port = self.master_port(name)
  517. - script = """
  518. -#!/bin/bash
  519. -usermod --uid {uid} jenkins
  520. -groupmod --gid {docker_group_id} docker
  521. -apt-get install -y lxc-docker-{docker_host_version}
  522. -su - -c '{tunnel}' jenkins
  523. -/usr/sbin/sshd -D -E /var/log/auth.log
  524. - """.format(tunnel=self.tunnel(master_port),
  525. - docker_group_id=grp.getgrnam('docker').gr_gid,
  526. - uid=os.getuid(),
  527. - docker_host_version=docker_host_version)
  528. - open(self.args.jenkins_home + '/setup.sh', 'w').write(script)
  529. - self.fetch_ssh_key(self.args.jenkins_home)
  530. - util.sh('docker stop ' + name + ' || true')
  531. - util.sh('docker rm ' + name + ' || true')
  532. - home = os.path.abspath(self.args.jenkins_home)
  533. - util.sh('docker run -d --name ' + name + ' ' +
  534. - ' -p ' + slave_port + ':22 ' +
  535. - ' -v ' + home + ':/home/jenkins ' +
  536. - ' -v /var/run/docker.sock:/run/docker.sock ' +
  537. - self.args.jenkins_slave + ' ' +
  538. - ' bash /home/jenkins/setup.sh')
  539. - return name
  540. -
  541. - def docker_build_master(self):
  542. - if self.args.jenkins_master not in util.sh('docker images'):
  543. - util.sh('docker build -t ' + self.args.jenkins_master + " " +
  544. - self.args.libdir + '/jenkins-docker')
  545. - return True
  546. - return False
  547. -
  548. - def run_master(self):
  549. - self.docker_build_master()
  550. - home = os.path.abspath(self.args.jenkins_home)
  551. - dotssh = home + '/.ssh'
  552. - if not os.path.exists(dotssh):
  553. - os.makedirs(dotssh, 0o700)
  554. - util.sh('ssh-keygen -N "" -f ' + dotssh + '/id_rsa')
  555. - shutil.copyfile(dotssh + '/id_rsa.pub',
  556. - dotssh + '/authorized_keys')
  557. - os.chmod(dotssh + '/authorized_keys', 0o600)
  558. - util.sh('docker run -d --name ' + self.args.jenkins_master +
  559. - ' -p 50000:50000' +
  560. - ' -p ' + self.args.jenkins_port + ':8080' +
  561. - ' -p ' + self.args.jenkins_ssh_port + ':22' +
  562. - ' -v ' + home + ':/var/jenkins_home' +
  563. - ' ' + self.args.jenkins_master)
  564. -
  565. - def wait_for_credentials():
  566. - timeout = 120
  567. - for t in range(timeout):
  568. - try:
  569. - self.add_credential()
  570. - logging.debug('run_master:wait_for_credentials ' +
  571. - str(self.slave_credential_id()))
  572. - logging.debug('run_master:wait_for_credentials done')
  573. - return
  574. - except requests.exceptions.ConnectionError as e:
  575. - logging.debug('run_master:wait_for_credentials ' + str(e))
  576. - except requests.exceptions.HTTPError as e:
  577. - logging.debug('run_master:wait_for_credentials ' + str(e))
  578. - except NoCredentials as e:
  579. - logging.debug('run_master:wait_for_credentials ' + str(e))
  580. - time.sleep(1)
  581. - raise Exception('failed to add credential')
  582. - wait_for_credentials()
  583. -
  584. - def destroy_master(self):
  585. - util.sh('docker stop ' + self.args.jenkins_master)
  586. - util.sh('docker rm ' + self.args.jenkins_master)
  587. - return True
  588. -
  589. - def add_credential(self):
  590. - cred = (cred_template.
  591. - replace('{KEY_FILE}', '/var/jenkins_home/.ssh/id_rsa').
  592. - replace('{USERNAME}', 'jenkins').
  593. - replace('{DESCRIPTION}', 'for jenkins slaves'))
  594. - r = requests.post(self.get_url() + '/credentials/configSubmit',
  595. - data={'description': 'GLOBAL DESCRIPTION',
  596. - 'name': 'GLOBAL NAME',
  597. - 'json': cred})
  598. - r.raise_for_status()
  599. - return r.text
  600. diff --git a/ceph_workbench/main.py b/ceph_workbench/main.py
  601. index a3c2084..5fa00a0 100644
  602. --- a/ceph_workbench/main.py
  603. +++ b/ceph_workbench/main.py
  604. @@ -18,7 +18,6 @@
  605. # along with this program. If not, see `<http://www.gnu.org/licenses/>`.
  606. #
  607. import argparse
  608. -from ceph_workbench import jenkins
  609. from ceph_workbench import wbbackport
  610. from github2gitlab import main
  611. import logging
  612. @@ -71,15 +70,6 @@ class CephWorkbench:
  613. func=wbbackport.WBBackport,
  614. )
  616. - subparsers.add_parser(
  617. - 'jenkins',
  618. - help='Drive and inspect the Jenkins building Ceph',
  619. - parents=[jenkins.Jenkins.get_parser()],
  620. - add_help=False,
  621. - ).set_defaults(
  622. - func=jenkins.Jenkins,
  623. - )
  624. -
  625. def run(self, argv):
  626. args = self.parser.parse_args(argv)
  628. diff --git a/data_files/jenkins-docker/Dockerfile b/data_files/jenkins-docker/Dockerfile
  629. deleted file mode 100644
  630. index 799b046..0000000
  631. --- a/data_files/jenkins-docker/Dockerfile
  632. +++ /dev/null
  633. @@ -1,13 +0,0 @@
  634. -FROM jenkins
  635. -USER root
  636. -RUN apt-get update && apt-get install -y openssh-server
  637. -# Standard SSH port
  638. -EXPOSE 22
  639. -EXPOSE 50000
  640. -
  641. -RUN sed -i 's|session required pam_loginuid.so|session optional pam_loginuid.so|g' /etc/pam.d/sshd
  642. -RUN mkdir -p /var/run/sshd
  643. -
  644. -RUN ( echo '#!/bin/bash' ; echo /usr/sbin/sshd ; echo chown -R jenkins /var/jenkins_home/.ssh ; echo /usr/local/bin/jenkins.sh ) > /usr/local/bin/jenkins-and-ssh.sh ; chmod +x /usr/local/bin/jenkins-and-ssh.sh
  645. -
  646. -ENTRYPOINT ["/usr/local/bin/jenkins-and-ssh.sh"]
  647. diff --git a/data_files/jenkins-slave-docker/Dockerfile b/data_files/jenkins-slave-docker/Dockerfile
  648. deleted file mode 100644
  649. index 6179bd3..0000000
  650. --- a/data_files/jenkins-slave-docker/Dockerfile
  651. +++ /dev/null
  652. @@ -1,32 +0,0 @@
  653. -# This Dockerfile is used to build an image containing basic stuff to be used as a Jenkins slave build node.
  654. -FROM ubuntu:trusty
  655. -MAINTAINER Loic Dachary <[email protected]>
  656. -
  657. -RUN echo deb http://get.docker.com/ubuntu docker main | tee /etc/apt/sources.list.d/docker.list ; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
  658. -# Make sure the package repository is up to date.
  659. -RUN apt-get update
  660. -RUN apt-get -y upgrade
  661. -
  662. -RUN apt-get install -y openssh-server git lxc-docker ntp python-pip jq ccache
  663. -# http://stackoverflow.com/questions/27341064/how-do-i-fix-importerror-cannot-import-name-incompleteread
  664. -RUN easy_install -U pip
  665. -RUN pip install ceph-workbench
  666. -RUN sed -i 's|session required pam_loginuid.so|session optional pam_loginuid.so|g' /etc/pam.d/sshd
  667. -RUN mkdir -p /var/run/sshd
  668. -
  669. -# Install JDK 7 (latest edition)
  670. -RUN apt-get install -y openjdk-7-jdk
  671. -
  672. -# Add user jenkins to the image
  673. -RUN adduser --quiet jenkins
  674. -# Set password for the jenkins user (you may want to alter this).
  675. -RUN echo "jenkins:jenkins" | chpasswd
  676. -
  677. -RUN usermod -aG docker jenkins
  678. -
  679. -RUN echo 'jenkins ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/jenkins
  680. -
  681. -# Standard SSH port
  682. -EXPOSE 22
  683. -
  684. -CMD ["/usr/sbin/sshd", "-D"]
  685. diff --git a/docs/index.rst b/docs/index.rst
  686. index 853fc6b..f6ee366 100644
  687. --- a/docs/index.rst
  688. +++ b/docs/index.rst
  689. @@ -25,11 +25,10 @@ development workflow for `Ceph <http://ceph.com>`_.
  690. subcommands:
  691. valid subcommands
  693. - {github2gitlab,backport,jenkins}
  694. + {github2gitlab,backport}
  695. sub-command -h
  696. github2gitlab Mirror a GitHub project to GitLab
  697. backport Backport reports
  698. - jenkins Drive and inspect the Jenkins building Ceph
  700. User Guide
  701. ----------
  702. @@ -43,7 +42,6 @@ instructions for getting the most out of ceph-workbench.
  704. user/intro
  705. user/install
  706. - user/jenkins
  709. Contributor Guide
  710. diff --git a/docs/user/jenkins.rst b/docs/user/jenkins.rst
  711. deleted file mode 100644
  712. index cc323e7..0000000
  713. --- a/docs/user/jenkins.rst
  714. +++ /dev/null
  715. @@ -1,152 +0,0 @@
  716. -.. _jenkins:
  717. -
  718. -Jenkins
  719. -=======
  720. -
  721. -::
  722. -
  723. - +----------------+
  724. - | commit pushed |
  725. - | in ceph |
  726. - +----------------+
  727. - | (0)
  728. - v
  729. - +----------------+
  730. - ssh tunnel | jenkins master | ssh tunnel
  731. - +-------------| ceph job |----------------+
  732. - | (1) +----------------+ |
  733. - | |
  734. - | |
  735. - v v
  736. - +----------+ +----------+
  737. - | jenkins | | jenkins |
  738. - | slave | | slave |
  739. - | (docker) | | (qemu) |
  740. - +----------+ +----------+
  741. - | | (3) |
  742. - (2) | +-------------------------+ (3) |
  743. - | | |
  744. - v v v
  745. - +-----------+ +--------------+ +--------------+
  746. - | container | | container | | qemu i386 |
  747. - | centos 7 | | ubuntu 14.04 | | ubuntu 14.04 |
  748. - +-----------+ +--------------+ +--------------+
  749. - | | |
  750. - v v |
  751. - +-------------------+ +-------------------+ |
  752. - | run-make-check.sh | | run-make-check.sh | |
  753. - +-------------------+ +-------------------+ |
  754. - | | v
  755. - | | +-------------------+
  756. - | +-----------------+ | | run-make-check.sh |
  757. - +---->| jenkins master |<---+ +-------------------+
  758. - | ceph-report | |
  759. - | job |<----------------------+
  760. - +-----------------+
  761. - |
  762. - | (4)
  763. - v
  764. -
  765. -
  766. -.. _`jenkins master`: http://jenkins.ceph.dachary.org/
  767. -.. _`jenkins job`: http://jenkins.ceph.dachary.org/job/ceph/
  768. -.. _`run-make-check.sh`: http://workbench.dachary.org/ceph/ceph/blob/master/run-make-check.sh
  769. -.. _`ceph repository`: http://workbench.dachary.org/ceph/ceph/
  770. -
  771. -A `jenkins job`_ is triggered every time a commit is pushed to the
  772. -`ceph repository`_ and executes the `run-make-check.sh`_ script to
  773. -verify it compiles and passes unit and functional tests.
  774. -
  775. -The `jenkins master`_ delegates the execution of `run-make-check.sh`_
  776. -to `jenkins slaves <http://jenkins.ceph.dachary.org/computer/>`_. If
  777. -the git branch matches a pull request found in the `Ceph official
  778. -repository <http://github.com/ceph/ceph/>`_, a comment is added when
  779. -the run is complete. When there is a failure the full output of the
  780. -script can be browsed to figure out what went wrong.
  781. -
  782. -Jenkins slave CPU & RAM requirements
  783. -------------------------------------
  784. -
  785. -The minimum configuration of a slave is:
  786. -
  787. -- bare metal with 32GB RAM, SSD with 16GB free space, 8 cores OR
  788. -- virtual machine with 32GB RAM, 20GB root disk, 8 cores
  789. -
  790. -It can be hosted on a private IP: it will initiate a tunnel to the
  791. -jenkins master.
  792. -
  793. -Hosting a jenkins slave
  794. ------------------------
  795. -
  796. -In the simplest way to contribute a jenkins slave is when a jenkins
  797. -admin can ssh to it as root. If the slave is behind a firewall
  798. -hosting_private it :ref:`requires two more steps <hosting_private>`.
  799. -
  800. -#. Add the following public key to the ``/root/.ssh/authorized_keys`` file
  801. - of the machine hosting the jenkins slave.
  802. -
  803. - ::
  804. -
  805. - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9mo/xFNhLSCADCjmPQ0MnlTdpIaC8O6vHT5/z7i0boZhLsTe84Gq6sg6qIQY/847/FX52wN6YxMYYjr4478eGa1n84L88WHA4updDT4/LbKzu3aYOVPD6NlkMGKmoJOQazY5z2Mpa/gYHDboZgyyLQ3ApSlM9SCc0xJIJwhGv4uAPjWDzkjCyMCjAOu0NPzJ97uuKgS7e5u1vxL3+6hn7HlIU9wSnA3PEmMUHC8p3f0sHnX5OeZcLwxOAD8v3Q74Lg7yNTc8K0wBuA/G32Tad2QZxFsuSkOyuvhQnWe7dqPL6Jvr20A9wu2A7WVbQ7YaxwngMjB26Pezxg4mFUYV ubuntu@jenkins
  806. -#. Send a mail to `Ceph Development
  807. - <mailto:[email protected]>`_ with the IP of the machine
  808. -
  809. -The jenkins admin will then do the following:
  810. -
  811. -#. ssh -A ``[email protected]``
  812. -#. Create a jenkins user with a home at ``/home/jenkins``
  813. -#. Allow the jenkins user to use docker
  814. -
  815. - ::
  816. -
  817. - $ sudo usermod -aG docker jenkins
  818. -
  819. -#. :ref:`Install <install>` ceph-workbench
  820. -#. Logout
  821. -#. ssh -A ``[email protected]``
  822. -#. Create a new node on the jenkins master, create a jenkins slave container and connect it with the new node.
  823. -
  824. - ::
  825. -
  826. - $ ceph-workbench jenkins \
  827. - --jenkins-url http://jenkins.ceph.dachary.org/ \
  828. - --jenkins-user admin \
  829. - --jenkins-password XXXXXXXX \
  830. - --jenkins-ssh-host [email protected] \
  831. - --jenkins-docker-run-slave new
  832. - slave940
  833. -#. Verify that the newly created node is connected and ready to run
  834. - jobs at http://jenkins.ceph.dachary.org/computer/slave094/
  835. -
  836. -.. _hosting_private:
  837. -
  838. -Hosting a jenkins slave on a private IP
  839. ----------------------------------------
  840. -
  841. -The process for hosting a jenkins slave requires more steps if the
  842. -hosted slave cannot be access from the net:
  843. -
  844. -#. Send a mail to `Ceph Development
  845. - <mailto:[email protected]>`_ with the configuration (RAM,
  846. - disk, CPU) of the machine that could host a jenkins slave.
  847. -#. Wait for the following to be sent:
  848. - - a ssh key
  849. - - the name of the slave (for instance **slave904**)
  850. -#. Create a jenkins user with a home at ``/home/jenkins``
  851. -#. Allow the jenkins user to use docker
  852. -
  853. - ::
  854. -
  855. - sudo usermod -aG docker jenkins
  856. -
  857. -#. :ref:`Install <install>` ceph-workbench
  858. -#. Login as jenkins
  859. -#. Add the ssh key to ``~/.ssh``
  860. -#. Connect to the jenkins master
  861. -
  862. - ::
  863. -
  864. - ceph-workbench jenkins \
  865. - --jenkins-url http://jenkins.ceph.dachary.org/ \
  866. - --jenkins-ssh-host [email protected] \
  867. - --jenkins-docker-run-slave slave904
  868. diff --git a/requirements.txt b/requirements.txt
  869. index 3af59f9..2f75c77 100644
  870. --- a/requirements.txt
  871. +++ b/requirements.txt
  872. @@ -7,3 +7,4 @@ github2gitlab
  873. six>=1.9.0
  874. sphinx>=1.1.3
  875. docutils>=0.11
  876. +twisted>=15.2
  877. diff --git a/tests/jenkins-define-credentials.py b/tests/jenkins-define-credentials.py
  878. deleted file mode 100644
  879. index 66a79f2..0000000
  880. --- a/tests/jenkins-define-credentials.py
  881. +++ /dev/null
  882. @@ -1,46 +0,0 @@
  883. -import requests
  884. -import sys
  885. -
  886. -cred_template = """
  887. -{
  888. - "domainCredentials": {
  889. - "domain": {
  890. - "name": "",
  891. - "description": ""
  892. - },
  893. - "credentials": [
  894. - {
  895. - "scope": "GLOBAL",
  896. - "id": "",
  897. - "username": "{USERNAME}",
  898. - "description": "{DESCRIPTION}",
  899. - "privateKeySource": {
  900. - "value": "1",
  901. - "privateKeyFile": "{KEY_FILE}",
  902. - "stapler-class": "com.cloudbees.jenkins.plugins\
  903. -.sshcredentials.impl.BasicSSHUserPrivateKey$FileOnMasterPrivateKeySource"
  904. - },
  905. - "passphrase": "",
  906. - "stapler-class": "com.cloudbees.jenkins.plugins\
  907. -.sshcredentials.impl.BasicSSHUserPrivateKey",
  908. - "kind": "com.cloudbees.jenkins.plugins\
  909. -.sshcredentials.impl.BasicSSHUserPrivateKey"
  910. - }
  911. - ]
  912. - }
  913. -}
  914. -"""
  915. -
  916. -
  917. -def init(url):
  918. - cred = (cred_template.
  919. - replace('{KEY_FILE}', '/var/jenkins_home/.ssh/id_rsa').
  920. - replace('{USERNAME}', 'ubuntu').
  921. - replace('{DESCRIPTION}', 'for jenkins slaves'))
  922. - r = requests.post(url + '/credentials/configSubmit',
  923. - data={'description': 'GLOBAL DESCRIPTION',
  924. - 'name': 'GLOBAL NAME',
  925. - 'json': cred})
  926. - r.raise_for_status()
  927. -
  928. -init(*sys.argv[1:])
  929. diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py
  930. deleted file mode 100644
  931. index 2782ef9..0000000
  932. --- a/tests/test_jenkins.py
  933. +++ /dev/null
  934. @@ -1,510 +0,0 @@
  935. -# -*- mode: python; coding: utf-8 -*-
  936. -#
  937. -# Copyright (C) 2015 <[email protected]>
  938. -#
  939. -# Author: Loic Dachary <[email protected]>
  940. -#
  941. -# This program is free software: you can redistribute it and/or modify
  942. -# it under the terms of the GNU General Public License as published by
  943. -# the Free Software Foundation, either version 3 of the License, or
  944. -# (at your option) any later version.
  945. -#
  946. -# This program is distributed in the hope that it will be useful,
  947. -# but WITHOUT ANY WARRANTY; without even the implied warranty of
  948. -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  949. -# GNU General Public License for more details.
  950. -#
  951. -# You should have received a copy of the GNU General Public License
  952. -# along with this program. If not, see <http://www.gnu.org/licenses/>.
  953. -#
  954. -import logging
  955. -import os
  956. -import responses
  957. -import shutil
  958. -import subprocess
  959. -import testtools
  960. -import threading
  961. -import time
  962. -
  963. -from ceph_workbench import jenkins
  964. -from ceph_workbench import util
  965. -
  966. -JENKINS = {
  967. - 'ssh-host': '[email protected]',
  968. - 'ssh-port': '5555',
  969. - 'key': 'data/jenkins/.ssh/id_rsa',
  970. - 'host': 'localhost',
  971. - 'port': '11080',
  972. - 'user': 'admin',
  973. - 'password': 'admin',
  974. - 'qemu-image': 'ubuntu-14.04-i386',
  975. -}
  976. -
  977. -logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
  978. - level=logging.DEBUG)
  979. -
  980. -
  981. -def try_a_few_times(command):
  982. - timeout = 120
  983. - for t in range(timeout):
  984. - time.sleep(1)
  985. - try:
  986. - return util.sh(command)
  987. - except subprocess.CalledProcessError:
  988. - pass
  989. - raise Exception(command + ' failed ' + str(timeout) + ' times')
  990. -
  991. -
  992. -class TestJenkinsAll(testtools.TestCase):
  993. -
  994. - @classmethod
  995. - def setUpClass(cls):
  996. - image_dir = 'data/' + JENKINS['qemu-image']
  997. - dotssh = image_dir + '/.ssh'
  998. - if os.path.exists(dotssh):
  999. - shutil.rmtree(dotssh)
  1000. - img = image_dir + '/' + JENKINS['qemu-image'] + '.img'
  1001. - if os.path.exists(img):
  1002. - os.unlink(img)
  1003. - home = 'data/jenkins'
  1004. - cls.j = jenkins.Jenkins.factory([
  1005. - '--jenkins-ssh-host', JENKINS['ssh-host'],
  1006. - '--jenkins-ssh-port', JENKINS['ssh-port'],
  1007. - '--jenkins-home', home,
  1008. - '--jenkins-host', JENKINS['host'],
  1009. - '--jenkins-port', JENKINS['port'],
  1010. - '--libdir', 'data_files',
  1011. - '--datadir', 'data',
  1012. - '--jenkins-run-master',
  1013. - ])
  1014. - cls.tearDownClass()
  1015. - cls.j.run()
  1016. - try_a_few_times(
  1017. - "ssh -p " + JENKINS['ssh-port'] +
  1018. - " -i " + JENKINS['key'] +
  1019. - " " + JENKINS['ssh-host'] +
  1020. - " ps fauwwwx")
  1021. - # when the cli port 50000 is not yet open, the cli returns a
  1022. - # rather confusing error message:
  1023. - # SEVERE: I/O error in channel Chunked connection ...
  1024. - # https://issues.jenkins-ci.org/browse/JENKINS-23232
  1025. -
  1026. - def wait_for_cli():
  1027. - timeout = 120
  1028. - for t in range(timeout):
  1029. - out = cls.j.run_cli('help')
  1030. - if "create-node" in out:
  1031. - return
  1032. - logging.debug('wait_for_cli ' + out)
  1033. - time.sleep(1)
  1034. - raise Exception('failed to communicate with cli')
  1035. - wait_for_cli()
  1036. -
  1037. - @classmethod
  1038. - def tearDownClass(cls):
  1039. - if os.path.exists(cls.j.get_cli_jar()):
  1040. - os.unlink(cls.j.get_cli_jar())
  1041. - util.sh('docker stop ' + cls.j.args.jenkins_master + ' || true')
  1042. - util.sh('docker rm ' + cls.j.args.jenkins_master + ' || true')
  1043. - util.sh('docker rmi ' + cls.j.args.jenkins_master + ' || true')
  1044. - util.sh('sudo rm -fr data/jenkins')
  1045. -
  1046. - @responses.activate
  1047. - def test_publish_build_to_pull_request(self):
  1048. - pr = '123'
  1049. - build = 'ceph/3'
  1050. - jenkins_url = 'http://jenkins'
  1051. - upstream_build = '12'
  1052. - upstream_url = 'job/ceph'
  1053. - build_report = """
  1054. -{
  1055. - "actions": [
  1056. - {
  1057. - "causes": [
  1058. - {
  1059. - "shortDescription": "Started by upstream project build number 12",
  1060. - "upstreamBuild": {upstream_build},
  1061. - "upstreamProject": "ceph",
  1062. - "upstreamUrl": "{upstream_url}"
  1063. - }
  1064. - ]
  1065. - },
  1066. - {}
  1067. - ],
  1068. - "artifacts": [],
  1069. - "building": false,
  1070. - "description": null,
  1071. - "duration": 3996,
  1072. - "estimatedDuration": 2042,
  1073. - "executor": null,
  1074. - "fullDisplayName": "ceph-report #1",
  1075. - "id": "2015-04-06_08-26-47",
  1076. - "keepLog": false,
  1077. - "number": 1,
  1078. - "result": "SUCCESS",
  1079. - "timestamp": 1428308807127,
  1080. - "url": "http://jenkins/job/ceph-report/1/",
  1081. - "builtOn": "rex003",
  1082. - "changeSet": {
  1083. - "items": [],
  1084. - "kind": null
  1085. - },
  1086. - "culprits": []
  1087. -}
  1088. -""".replace('{upstream_url}',
  1089. - upstream_url).replace('{upstream_build}',
  1090. - upstream_build)
  1091. - responses.add(responses.GET, jenkins_url + '/' + build + '/api/json',
  1092. - body=build_report, status=200,
  1093. - content_type='application/json')
  1094. - os_url = 'http://jenkins/job/ceph/TARGET_OPERATING_SYSTEM'
  1095. - upstream_build_report = """
  1096. -{
  1097. - "actions": [
  1098. - {
  1099. - "causes": [
  1100. - {
  1101. - "shortDescription": "Started by user Ceph Admin",
  1102. - "userId": "admin",
  1103. - "userName": "Ceph Admin"
  1104. - }
  1105. - ]
  1106. - },
  1107. - {},
  1108. - {
  1109. - "buildsByBranchName": {
  1110. - "refs/remotes/origin/pull/123": {
  1111. - "buildNumber": 12,
  1112. - "buildResult": null,
  1113. - "marked": {
  1114. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1115. - "branch": [
  1116. - {
  1117. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1118. - "name": "refs/remotes/origin/pull/123"
  1119. - }
  1120. - ]
  1121. - },
  1122. - "revision": {
  1123. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1124. - "branch": [
  1125. - {
  1126. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1127. - "name": "refs/remotes/origin/pull/123"
  1128. - }
  1129. - ]
  1130. - }
  1131. - }
  1132. - },
  1133. - "lastBuiltRevision": {
  1134. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1135. - "branch": [
  1136. - {
  1137. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1138. - "name": "refs/remotes/origin/pull/123"
  1139. - }
  1140. - ]
  1141. - },
  1142. - "remoteUrls": [
  1143. - "http://workbench.dachary.org/ceph/ceph.git"
  1144. - ],
  1145. - "scmName": ""
  1146. - },
  1147. - {},
  1148. - {}
  1149. - ],
  1150. - "artifacts": [],
  1151. - "building": false,
  1152. - "description": null,
  1153. - "duration": 31117,
  1154. - "estimatedDuration": 1971442,
  1155. - "executor": null,
  1156. - "fullDisplayName": "ceph #12",
  1157. - "id": "2015-04-06_08-26-06",
  1158. - "keepLog": false,
  1159. - "number": 12,
  1160. - "result": "FAILURE",
  1161. - "timestamp": 1428308766532,
  1162. - "url": "http://jenkins/{upstream_url}/{upstream_build}/",
  1163. - "builtOn": "rex003",
  1164. - "changeSet": {
  1165. - "items": [],
  1166. - "kind": "git"
  1167. - },
  1168. - "culprits": [],
  1169. - "runs": [
  1170. - {
  1171. - "number": {upstream_build},
  1172. - "url": "{os_url}=centos-7/{upstream_build}/"
  1173. - },
  1174. - {
  1175. - "number": {upstream_build},
  1176. - "url": "{os_url}=ubuntu-14.04/{upstream_build}/"
  1177. - },
  1178. - {
  1179. - "number": 1,
  1180. - "url": "http://jenkins/job/ceph/system=centos-7/1/"
  1181. - }
  1182. - ]
  1183. -}
  1184. -""".replace('{upstream_url}',
  1185. - upstream_url).replace('{upstream_build}',
  1186. - upstream_build).replace('{os_url}', os_url)
  1187. - responses.add(responses.GET, jenkins_url + '/' +
  1188. - upstream_url + '/' + upstream_build + '/api/json',
  1189. - body=upstream_build_report, status=200,
  1190. - content_type='application/json')
  1191. - build_url = ("http://jenkins/" + upstream_url +
  1192. - "/TARGET_OPERATING_SYSTEM=centos-7/" + upstream_build)
  1193. - os_build_report = """
  1194. -{
  1195. - "actions": [
  1196. - {
  1197. - "lastBuiltRevision": {
  1198. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1199. - "branch": [
  1200. - {
  1201. - "SHA1": "23549bfbbdc88c874d9660889450612d2d372b26",
  1202. - "name": "refs/remotes/origin/pull/123"
  1203. - }
  1204. - ]
  1205. - }
  1206. - }
  1207. - ],
  1208. - "number": {upstream_build},
  1209. - "result": "{result}",
  1210. - "url": "{build_url}"
  1211. -}
  1212. -"""
  1213. - centos_build_report = (os_build_report.
  1214. - replace('{build_url}', build_url).
  1215. - replace('{result}', 'SUCCESS').
  1216. - replace('{upstream_build}', upstream_build))
  1217. - responses.add(responses.GET, build_url + '/api/json',
  1218. - body=centos_build_report, status=200,
  1219. - content_type='application/json')
  1220. - build_url = ("http://jenkins/" + upstream_url +
  1221. - "/TARGET_OPERATING_SYSTEM=ubuntu-14.04/" +
  1222. - upstream_build)
  1223. - ubuntu_build_report = (os_build_report.
  1224. - replace('{build_url}', build_url).
  1225. - replace('{result}', 'FAILURE').
  1226. - replace('{upstream_build}', upstream_build))
  1227. - responses.add(responses.GET, build_url + '/api/json',
  1228. - body=ubuntu_build_report, status=200,
  1229. - content_type='application/json')
  1230. - github_repo = 'my/repo'
  1231. - github_token = 'TOKEN'
  1232. - expected = """\
  1233. -FAILURE: http://jenkins/job/ceph/12/
  1234. -* SUCCESS http://jenkins/job/ceph/TARGET_OPERATING_SYSTEM=centos-7/12/
  1235. -* FAILURE http://jenkins/job/ceph/TARGET_OPERATING_SYSTEM=ubuntu-14.04/12/
  1236. -"""
  1237. - github_url = ('https://api.github.com/repos/' + github_repo +
  1238. - '/issues/' + pr + '/comments?access_token=' +
  1239. - github_token)
  1240. -
  1241. - def request_callback(request):
  1242. - return (200, {}, request.body)
  1243. - responses.add_callback(responses.POST, github_url,
  1244. - callback=request_callback,
  1245. - content_type='application/json',
  1246. - match_querystring=True)
  1247. - j = jenkins.Jenkins.factory([
  1248. - '--publish-build-to-pull-request', build,
  1249. - '--jenkins-host', JENKINS['host'],
  1250. - '--jenkins-port', JENKINS['port'],
  1251. - '--github-repo', github_repo,
  1252. - '--github-token', github_token,
  1253. - ])
  1254. - self.assertEquals(expected, j.run())
  1255. -
  1256. - def test_next_node_name(self):
  1257. - j = jenkins.Jenkins.factory([
  1258. - '--jenkins-host', JENKINS['host'],
  1259. - '--jenkins-port', JENKINS['port'],
  1260. - ])
  1261. - self.assertEquals("slave001", j.next_node_name())
  1262. -
  1263. - def test_slave_credential_id(self):
  1264. - j = jenkins.Jenkins.factory([
  1265. - '--jenkins-host', JENKINS['host'],
  1266. - '--jenkins-port', JENKINS['port'],
  1267. - '--jenkins-user', JENKINS['user'],
  1268. - '--jenkins-password', JENKINS['password'],
  1269. - ])
  1270. - self.assertTrue("-" in j.slave_credential_id())
  1271. -
  1272. - def test_run_cli(self):
  1273. - j = jenkins.Jenkins.factory([
  1274. - '--jenkins-host', JENKINS['host'],
  1275. - '--jenkins-port', JENKINS['port'],
  1276. - ])
  1277. - self.assertIn("create-node", j.run_cli('help'))
  1278. -
  1279. - def test_create_node(self):
  1280. - j = jenkins.Jenkins.factory([
  1281. - '--jenkins-host', JENKINS['host'],
  1282. - '--jenkins-port', JENKINS['port'],
  1283. - '--jenkins-user', JENKINS['user'],
  1284. - '--jenkins-password', JENKINS['password'],
  1285. - ])
  1286. - name = 'NAME005'
  1287. - n = j.create_node(name)
  1288. - self.assertEquals(name, n['displayName'])
  1289. - j.destroy_node(name)
  1290. -
  1291. - def test_build_slave(self):
  1292. - j = jenkins.Jenkins.factory([
  1293. - '--libdir', 'data_files',
  1294. - ])
  1295. - j.docker_build_slave()
  1296. - self.assertIn(j.args.jenkins_slave,
  1297. - util.sh("docker images " + j.args.jenkins_slave))
  1298. -
  1299. - def test_build_libguestfs(self):
  1300. - j = jenkins.Jenkins.factory([
  1301. - '--libdir', 'data_files',
  1302. - ])
  1303. - j.docker_build_libguestfs()
  1304. - self.assertIn(j.args.libguestfs,
  1305. - util.sh("docker images " + j.args.libguestfs))
  1306. -
  1307. - def test_prepare_qemu_img(self):
  1308. - j = jenkins.Jenkins.factory([
  1309. - '--jenkins-ssh-host', JENKINS['ssh-host'],
  1310. - '--jenkins-ssh-key', JENKINS['key'],
  1311. - '--jenkins-ssh-port', JENKINS['ssh-port'],
  1312. - '--jenkins-qemu-image', JENKINS['qemu-image'],
  1313. - '--libdir', 'data_files',
  1314. - '--datadir', 'data',
  1315. - '--jenkins-qemu-prepare',
  1316. - ])
  1317. - j.run()
  1318. - self.assertTrue(os.path.exists('data/' + JENKINS['qemu-image']))
  1319. - self.assertTrue(os.path.exists('data/' + JENKINS['qemu-image'] +
  1320. - '/.ssh/id_rsa'))
  1321. -
  1322. - def test_qemu_run_jenkins_slave(self):
  1323. - options = [
  1324. - '--jenkins-ssh-host', JENKINS['ssh-host'],
  1325. - '--jenkins-ssh-key', JENKINS['key'],
  1326. - '--jenkins-ssh-port', JENKINS['ssh-port'],
  1327. - '--jenkins-host', JENKINS['host'],
  1328. - '--jenkins-port', JENKINS['port'],
  1329. - '--jenkins-user', JENKINS['user'],
  1330. - '--jenkins-password', JENKINS['password'],
  1331. - '--jenkins-qemu-image', JENKINS['qemu-image'],
  1332. - '--libdir', 'data_files',
  1333. - '--datadir', 'data',
  1334. - ]
  1335. - j = jenkins.Jenkins.factory(options)
  1336. - name = j.next_node_name()
  1337. - master_port = str(j.master_port(name))
  1338. - slave_port = str(j.slave_port(name))
  1339. - # this is implicit when calling qemu_run_slave
  1340. - # but we call it outside of the thread so that it
  1341. - # blocks the test because it may take a long time
  1342. - # and be confused with something going wrong in
  1343. - # the thread
  1344. - j.qemu_get_img()
  1345. -
  1346. - def kvm_start():
  1347. - other = jenkins.Jenkins.factory(options + [
  1348. - '--jenkins-qemu-run-slave', 'new',
  1349. - ])
  1350. - other.qemu_run_slave(name)
  1351. - kvm = threading.Thread(target=kvm_start)
  1352. - kvm.start()
  1353. -
  1354. - known_hosts = j.ignore_known_hosts()
  1355. - # ssh to the slave and give time to kvm launch
  1356. - via_slave_cmdline = try_a_few_times(
  1357. - "ssh -p " + slave_port +
  1358. - " " + known_hosts +
  1359. - " -i " + JENKINS['key'] +
  1360. - " jenkins@localhost cat /proc/cmdline")
  1361. - self.assertIn(master_port, via_slave_cmdline)
  1362. - util.sh("ssh -p " + slave_port +
  1363. - " " + known_hosts +
  1364. - " -i " + JENKINS['key'] +
  1365. - " jenkins@localhost ps fauwwx")
  1366. - # ssh to the slave via the master
  1367. - via_master_cmdline = try_a_few_times(
  1368. - "ssh -p " + JENKINS['ssh-port'] +
  1369. - " " + known_hosts +
  1370. - " -i " + JENKINS['key'] +
  1371. - " " + JENKINS['ssh-host'] +
  1372. - " ssh -p " + master_port +
  1373. - " " + known_hosts +
  1374. - " localhost cat /proc/cmdline")
  1375. - self.assertIn('ds=nocloud-net', via_master_cmdline)
  1376. - # halt the kvm process and expect the kvm thread to return
  1377. - j.qemu_destroy_slave(name)
  1378. - kvm.join()
  1379. - self.assertFalse(j.find_node(name))
  1380. -
  1381. - def test_docker_run_jenkins_slave(self):
  1382. - options = [
  1383. - '--jenkins-ssh-host', JENKINS['ssh-host'],
  1384. - '--jenkins-ssh-key', JENKINS['key'],
  1385. - '--jenkins-ssh-port', JENKINS['ssh-port'],
  1386. - '--jenkins-host', JENKINS['host'],
  1387. - '--jenkins-port', JENKINS['port'],
  1388. - '--jenkins-user', JENKINS['user'],
  1389. - '--jenkins-password', JENKINS['password'],
  1390. - '--libdir', 'data_files',
  1391. - '--jenkins-home', 'data_files',
  1392. - ]
  1393. - j = jenkins.Jenkins.factory(options + [
  1394. - '--jenkins-docker-run-slave', 'new',
  1395. - ])
  1396. - name = j.run()
  1397. - self.assertTrue(j.find_node(name))
  1398. - j.docker_destroy_slave(name)
  1399. - self.assertFalse(j.find_node(name))
  1400. -
  1401. - name = 'slave010'
  1402. - j = jenkins.Jenkins.factory(options + [
  1403. - '--jenkins-docker-run-slave', name,
  1404. - ])
  1405. - self.assertEquals(name, j.run())
  1406. - j.docker_destroy_slave(name)
  1407. - self.assertFalse(j.find_node(name))
  1408. -
  1409. -
  1410. -class TestJenkinsMaster(testtools.TestCase):
  1411. -
  1412. - def test_build_master(self):
  1413. - j = jenkins.Jenkins.factory([
  1414. - '--libdir', 'data_files',
  1415. - ])
  1416. - util.sh('docker rmi ' + j.args.jenkins_master + ' || true')
  1417. - j.docker_build_master()
  1418. - self.assertIn(j.args.jenkins_master,
  1419. - util.sh("docker images " + j.args.jenkins_master))
  1420. -
  1421. - def test_run_master(self):
  1422. - home = 'data/jenkins'
  1423. - j = jenkins.Jenkins.factory([
  1424. - '--jenkins-ssh-host', JENKINS['ssh-host'],
  1425. - '--jenkins-ssh-port', JENKINS['ssh-port'],
  1426. - '--jenkins-home', home,
  1427. - '--jenkins-host', JENKINS['host'],
  1428. - '--jenkins-port', JENKINS['port'],
  1429. - '--libdir', 'data_files',
  1430. - '--datadir', 'data',
  1431. - '--jenkins-run-master',
  1432. - ])
  1433. - util.sh('docker stop ' + j.args.jenkins_master + ' || true')
  1434. - util.sh('docker rm ' + j.args.jenkins_master + ' || true')
  1435. - util.sh('docker rmi ' + j.args.jenkins_master + ' || true')
  1436. - util.sh('sudo rm -fr data/jenkins')
  1437. - j.run()
  1438. - out = try_a_few_times(
  1439. - "ssh -p " + JENKINS['ssh-port'] +
  1440. - " -i " + JENKINS['key'] +
  1441. - " " + JENKINS['ssh-host'] +
  1442. - " ps fauwwwx")
  1443. - self.assertIn("jenkins.war", out)
  1444. - self.assertTrue(j.destroy_master())