Run bhyve guests under s6 supervision

roles/bhyve-s6/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reload s6-rc
+  service:
+    name: s6-rc
+    state: reloaded

roles/bhyve-s6/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: s6-rc

roles/bhyve-s6/tasks/main.yml

@@ -0,0 +1,94 @@
+---
+- name: Define bhyve guest service definitions
+  file:
+    path: '/etc/s6-rc/service/bhyve-{{ item.0.name }}{{ item.1 }}'
+    state: directory
+    owner: root
+    group: wheel
+    mode: 0755
+  loop_control:
+    label: 'bhyve-{{ item.0.name }}{{ item.1 }}'
+  with_nested:
+    - '{{ bhyve_guests }}'
+    - '{{ bhyve_dirs }}'
+  notify:
+    - Reload s6-rc
+
+- name: Instantiating service templates
+  template:
+    dest: '/etc/s6-rc/service/bhyve-{{ item.0.name }}{{ item.1.name }}'
+    src: 'bhyve{{ item.1.name }}.j2'
+    owner: root
+    group: wheel
+    mode: '{{ item.1.mode }}'
+  loop_control:
+    label: 'bhyve-{{ item.0.name }}{{ item.1.name }}'
+  with_nested:
+    - '{{ bhyve_guests }}'
+    - '{{ bhyve_templates }}'
+  notify:
+    - Reload s6-rc
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Start enabled bhyve guests
+  command: >
+    fdmove -c 2 1 s6-rc -v 2 -u change bhyve-{{ item.name }}
+  register: change
+  when: item.enabled | default
+  changed_when: change.stdout | length > 0
+  loop_control:
+    label: 'bhyve-{{ item.name }}'
+  with_items: '{{ bhyve_guests }}'
+
+- name: Create bhyve service
+  file:
+    path: '{{ item }}'
+    state: directory
+    owner: root
+    group: wheel
+    mode: 0755
+  notify:
+    - Reload s6-rc
+  with_items:
+    - /etc/s6-rc/service/bhyve
+    - /etc/s6-rc/service/bhyve-disabled
+    - /etc/s6-rc/service/bhyve-enabled
+
+- name: Declare bhyve service as bundle
+  copy:
+    dest: '/etc/s6-rc/service/{{ item }}/type'
+    content: bundle
+    owner: root
+    group: wheel
+    mode: 0444
+  notify:
+    - Reload s6-rc
+  with_items:
+    - bhyve
+    - bhyve-enabled
+    - bhyve-disabled
+
+- name: Define bhyve service bundles
+  template:
+    dest: '/etc/s6-rc/service/{{ item }}/contents'
+    src: '{{ item }}-contents.j2'
+    owner: root
+    group: wheel
+    mode: 0444
+  notify:
+    - Reload s6-rc
+  with_items:
+    - bhyve
+    - bhyve-enabled
+    - bhyve-disabled
+
+- name: Flush handlers (again)
+  meta: flush_handlers
+
+- name: Stop disabled bhyve guests
+  command: >
+    fdmove -c 2 1 s6-rc -v 2 -d change bhyve-disabled
+  register: change
+  changed_when: change.stdout | length > 0

roles/bhyve-s6/templates/bhyve-contents.j2

@@ -0,0 +1,4 @@
+{% for guest in bhyve_guests %}
+bhyve-{{ guest.name }}
+bhyve-{{ guest.name }}-log
+{% endfor %}

roles/bhyve-s6/templates/bhyve-disabled-contents.j2

@@ -0,0 +1,6 @@
+{% for guest in bhyve_guests %}
+{% if not (guest.enabled | default) %}
+bhyve-{{ guest.name }}
+bhyve-{{ guest.name }}-log
+{% endif %}
+{% endfor %}

roles/bhyve-s6/templates/bhyve-enabled-contents.j2

@@ -0,0 +1,6 @@
+{% for guest in bhyve_guests %}
+{% if guest.enabled | default %}
+bhyve-{{ guest.name }}
+bhyve-{{ guest.name }}-log
+{% endif %}
+{% endfor %}

roles/bhyve-s6/templates/bhyve-log/consumer-for.j2

@@ -0,0 +1 @@
+bhyve-{{ item.0.name }}

roles/bhyve-s6/templates/bhyve-log/env/DIR.j2

@@ -0,0 +1 @@
+/var/log/bhyve-{{ item.0.name }}

roles/bhyve-s6/templates/bhyve-log/env/GROUP.j2

@@ -0,0 +1 @@
+s6-log

roles/bhyve-s6/templates/bhyve-log/env/MODE.j2

@@ -0,0 +1 @@
+750

roles/bhyve-s6/templates/bhyve-log/env/NAME.j2

@@ -0,0 +1 @@
+{{ item.0.name }}

roles/bhyve-s6/templates/bhyve-log/env/PATH.j2

@@ -0,0 +1 @@
+/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin

roles/bhyve-s6/templates/bhyve-log/env/USER.j2

@@ -0,0 +1 @@
+s6-log

roles/bhyve-s6/templates/bhyve-log/finish.j2

@@ -0,0 +1,13 @@
+#!/usr/local/bin/execlineb -S2
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME NAME
+}
+
+fdmove -c 1 2
+ifelse { test "${1}" -eq 0 } {
+        echo "bhyve-${NAME}-log: Stopped."
+}
+        echo "bhyve-${NAME}-log: Failed with exit status (${1}, ${2})."

roles/bhyve-s6/templates/bhyve-log/notification-fd.j2

@@ -0,0 +1 @@
+3

roles/bhyve-s6/templates/bhyve-log/run.j2

@@ -0,0 +1,30 @@
+#!/usr/local/bin/execlineb -P
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+	importas -i -u NAME  NAME
+	importas -i -u USER  USER
+	importas -i -u GROUP GROUP
+	importas -i -u MODE  MODE
+	importas -i -u DIR   DIR
+}
+
+foreground { fdmove -c 1 2 echo "bhyve-${NAME}-log: Starting." }
+ 
+ifelse -n { install -d -o "${USER}" -g "${GROUP}" -m "${MODE}" "$DIR" } {
+ 	foreground { fdmove -c 1 2 echo "bhyve-${NAME}-log: Failed to create logging directory." }
+ 	false
+}
+ 
+ifelse -n { fdmove 1 3 echo } {
+        foreground { fdmove -c 1 2 echo "bhyve-${NAME}-log: Failed to signal readiness." }
+ 	false
+}
+ 
+foreground { fdmove -c 1 2 echo "bhyve-${NAME}-log: Ready." }
+ 
+fdmove -c 2 1
+ 
+s6-envuidgid $USER
+s6-log T $DIR

roles/bhyve-s6/templates/bhyve-log/type.j2

@@ -0,0 +1 @@
+longrun

roles/bhyve-s6/templates/bhyve/data/check.j2

@@ -0,0 +1,19 @@
+#!/usr/local/bin/execlineb -P
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME NAME
+	importas -i -u PORT PORT
+}
+
+ifelse {
+	redirfd -w 1 /dev/null
+	fdmove -c 2 1
+	test -c "/dev/vmm/${NAME}"
+} {
+	foreground { fdmove -c 1 2 echo "bhyve-${NAME}: Ready." }
+	true
+}
+	foreground { fdmove -c 1 2 echo "bhyve-${NAME}: Poll." }
+	false

roles/bhyve-s6/templates/bhyve/env/COM.j2

@@ -0,0 +1 @@
+nmdm{{ item.0.index }}A

roles/bhyve-s6/templates/bhyve/env/CPUS.j2

@@ -0,0 +1 @@
+{{ item.0.cpus }}

roles/bhyve-s6/templates/bhyve/env/DISKS.j2

@@ -0,0 +1,5 @@
+{% set disks = [] %}
+{% for disk in item.0.disks %}
+{{- disks.append("hd:/dev/zvol/"+bhyve_pool+"/bhyve/guests/"+item.0.name+"/"+disk.name) -}}
+{% endfor %}
+{{ disks | join(",") }}

roles/bhyve-s6/templates/bhyve/env/ISO.j2

@@ -0,0 +1 @@
+cd:/bhyve/images/{{ item.0.image }}

roles/bhyve-s6/templates/bhyve/env/NAME.j2

@@ -0,0 +1 @@
+{{ item.0.name }}

roles/bhyve-s6/templates/bhyve/env/NIC.j2

@@ -0,0 +1 @@
+vmnet{{ item.0.index }}

roles/bhyve-s6/templates/bhyve/env/ORDER.j2

@@ -0,0 +1 @@
+{{ item.0.order | default(bhyve_default_order) | join(" ") }}

roles/bhyve-s6/templates/bhyve/env/PASS.j2

@@ -0,0 +1 @@
+{{ item.0.password | default("") }}

roles/bhyve-s6/templates/bhyve/env/PATH.j2

@@ -0,0 +1 @@
+/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin

roles/bhyve-s6/templates/bhyve/env/PORT.j2

@@ -0,0 +1 @@
+{{ bhyve_port + item.0.index }}

roles/bhyve-s6/templates/bhyve/env/RAM.j2

@@ -0,0 +1 @@
+{{ item.0.ram }}

roles/bhyve-s6/templates/bhyve/env/ROM.j2

@@ -0,0 +1 @@
+/usr/local/share/uefi-firmware/BHYVE_UEFI.fd

roles/bhyve-s6/templates/bhyve/finish.j2

@@ -0,0 +1,28 @@
+#!/usr/local/bin/execlineb -S2
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME NAME
+}
+
+fdmove -c 1 2
+ifelse { test "${1}" -eq 0 } {
+	foreground { echo "bhyve-${NAME}: Guest requested a reboot." }
+	bhyvectl --vm="${NAME}" --destroy
+}
+ifelse { test "${1}" -eq 1 } {
+	foreground { echo "bhyve-${NAME}: Guest requested a power off." }
+	foreground { bhyvectl --vm="${NAME}" --destroy }
+	s6-svc -O .
+}
+ifelse { test "${1}" -eq 2 } {
+	foreground { echo "bhyve-${NAME}: Guest requested a halt." }
+	s6-svc -O .
+}
+ifelse { test "${1}" -eq 3 } {
+	foreground { echo "bhyve-${NAME}: Guest crashed with a triple fault. Rebooting." }
+	bhyvectl --vm="${NAME}" --destroy
+}
+foreground { echo "Guest ${NAME} caused bhyve to exit with (${1}, ${2})." }
+bhyvectl --vm="${NAME}" --destroy

roles/bhyve-s6/templates/bhyve/notification-fd.j2

@@ -0,0 +1 @@
+3

roles/bhyve-s6/templates/bhyve/producer-for.j2

@@ -0,0 +1 @@
+bhyve-{{ item.0.name }}-log

roles/bhyve-s6/templates/bhyve/run.j2

@@ -0,0 +1,53 @@
+#!/usr/local/bin/execlineb -P
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u    NAME  NAME
+	importas -i -u -s ORDER ORDER
+	importas -i -u    RAM   RAM
+	importas -i -u    ROM   ROM
+	importas -i -u    CPUS  CPUS
+	importas -i -u    NIC   NIC
+	importas -i -u    COM   COM
+	importas -i -u    PORT  PORT
+	importas -i -u    PASS  PASS
+}
+
+backtick -n AHCI {
+        forx X { $ORDER }
+		importas X X
+		importas Y $X
+		echo -n ,$Y
+}
+importas -i -u AHCI AHCI
+
+foreground { if { test -e "/dev/vmm/${NAME}" } bhyvectl --vm="${NAME}" --destroy }
+foreground { /etc/rc.d/netif start "${NIC}" }
+foreground { fdmove -c 1 2 echo "bhyve-${NAME}: Starting VM ${NAME} with ${CPUS} CPUs, ${RAM} RAM, COM on ${COM}, NIC ${NIC} and VNC port ${PORT}." }
+
+s6-notifyoncheck -d -w 100 -n 70
+
+fdmove -c 2 1
+# Use a static password to make VNC clients happy
+ifelse { test -n "$PASS" } {
+	bhyve -c "${CPUS}" -m "${RAM}" -w -A -P -H
+		-s "0,amd_hostbridge"
+		-s "2:0,ahci${AHCI}"
+		-s "3,virtio-net,${NIC}"
+		-s "4,virtio-rnd"
+		-s "29,fbuf,tcp=[::1]:${PORT},w=800,h=600,password=${PASS}"
+		-s "31,lpc"
+		-l "com1,/dev/${COM}"
+		-l "bootrom,${ROM}"
+	"${NAME}"
+}
+	bhyve -c "${CPUS}" -m "${RAM}" -w -A -P -H
+		-s "0,amd_hostbridge"
+		-s "2:0,ahci${AHCI}"
+		-s "3,virtio-net,${NIC}"
+		-s "4,virtio-rnd"
+		-s "31,lpc"
+		-l "com1,/dev/${COM}"
+		-l "bootrom,${ROM}"
+	"${NAME}"

roles/bhyve-s6/templates/bhyve/type.j2

@@ -0,0 +1 @@
+longrun

roles/bhyve-s6/vars/main.yml

@@ -0,0 +1,98 @@
+---
+bhyve_default_order:
+  - DISKS
+  - ISO
+bhyve_dirs:
+  - ''
+  - '/env'
+  - '/data'
+  - '-log'
+  - '-log/env'
+
+bhyve_templates:
+  - name: '/type'
+    mode: '0444'
+
+  - name: '/run'
+    mode: '0555'
+
+  - name: '/finish'
+    mode: '0555'
+
+  - name: '/producer-for'
+    mode: '0444'
+
+  - name: '/notification-fd'
+    mode: '0444'
+
+  - name: '/data/check'
+    mode: '0555'
+
+  - name: '/env/NAME'
+    mode: '0444'
+
+  - name: '/env/PATH'
+    mode: '0444'
+
+  - name: '/env/DISKS'
+    mode: '0444'
+
+  - name: '/env/ISO'
+    mode: '0444'
+
+  - name: '/env/ORDER'
+    mode: '0444'
+
+  - name: '/env/RAM'
+    mode: '0444'
+
+  - name: '/env/ROM'
+    mode: '0444'
+
+  - name: '/env/CPUS'
+    mode: '0444'
+
+  - name: '/env/NIC'
+    mode: '0444'
+
+  - name: '/env/COM'
+    mode: '0444'
+
+  - name: '/env/PORT'
+    mode: '0444'
+
+  - name: '/env/PASS'
+    mode: '0400'
+
+  - name: '-log/type'
+    mode: '0444'
+
+  - name: '-log/run'
+    mode: '0555'
+
+  - name: '-log/finish'
+    mode: '0555'
+
+  - name: '-log/consumer-for'
+    mode: '0444'
+
+  - name: '-log/notification-fd'
+    mode: '0444'
+
+  - name: '-log/env/NAME'
+    mode: '0444'
+
+  - name: '-log/env/PATH'
+    mode: '0444'
+
+  - name: '-log/env/USER'
+    mode: '0444'
+
+  - name: '-log/env/GROUP'
+    mode: '0444'
+
+  - name: '-log/env/MODE'
+    mode: '0444'
+
+  - name: '-log/env/DIR'
+    mode: '0444'