add cockroachdb support

defaults/main.yml

@@ -1,32 +1,22 @@
 ---
 osp_git_root: 'https://gitlab.com/Deamos/flask-nginx-rtmp-manager.git'
 osp_git_commit: '0.7.9'
-osp_worker_rtmp_port: 5999
+osp_worker_start_port: 5000
-osp_worker_start_port: 5010
 osp_worker_count: "{{ ansible_processor_nproc }}"
-osp_worker_timeout: 30
-# should not be edited
-osp_worker_worker: 1
-
-osp_tls_enabled: False
-osp_hostname: "{{ inventory_hostname }}"
 
 osp_http_user: "http"
 osp_http_group: "http"
-osp_http_path: "/var/www"
+osp_http_path: "/srv/http"
 
-osp_db_host: localhost
 osp_db_name: osp
 osp_db_user: osp
 osp_db_location: 'sqlite:///db/database.db'
 osp_db_type: "sqlite"
-#osp_db_pass: "" # required
 #osp_db_type: "cockroachdb"
-osp_db_location_postgresql: "postgresql://{{ osp_db_user }}@{{ osp_db_host }}/{{ osp_db_name }}"
+osp_db_location_cockroachdb: "postgresql://{{ osp_db_user }}@localhost:26257/{{ osp_db_name }}?sslmode=require&sslrootcert=/etc/cockroach/ca.crt&sslcert=/etc/cockroach/client.{{ osp_db_user }}.crt&sslkey=/etc/cockroach/client.{{ osp_db_user }}.key"
-osp_db_location_mysql: "mysql+pymysql://{{ osp_db_user }}:{{osp_db_pass}}@{{ osp_db_host }}/{{ osp_db_name }}?charset=utf8mb4"
 osp_secret_key: "{{ lookup('password', 'credentials/'+inventory_hostname+'/osp_secret_key length=8 chars=digits') }}"
 osp_password_salt: "{{ lookup('password', 'credentials/'+inventory_hostname+'/osp_password_salt length=8 chars=digits') }}"
 osp_allow_registration: yes
 osp_require_email_registration: yes
-osp_ejabberd_domain: "{{ osp_hostname }}"
+osp_ejabberd_domain: "CHANGEME"
 osp_ejabberd_password: "{{ lookup('password', 'credentials/'+inventory_hostname+'/osp_ejabberd_password length=8 chars=digits') }}"

handlers/main.yml

@@ -14,17 +14,8 @@
     name: nginx
     state: reloaded
 
-- name: dehydrated
-  command: /usr/bin/dehydrated -c
-
 - name: restart osp
   systemd:
     name: osp.target
     state: restarted
     daemon_reload: yes
-
-- name: restart osp-rtmp
-  systemd:
-    name: osp-rtmp
-    state: restarted
-    daemon_reload: yes

tasks/db_cockroachdb.yml

@@ -0,0 +1,46 @@
+- name: Install dependencies
+  package:
+    name: postgresql-libs
+
+- name: Install python cockroach requirements
+  pip:
+    virtualenv: /opt/osp-venv
+    name:
+      - sqlalchemy-cockroachdb
+
+- name: Create user
+  become: yes
+  command:
+    argv:
+      - cockroach
+      - sql
+      - --certs-dir=/etc/cockroach
+      - -e
+      - "CREATE DATABASE IF NOT EXISTS {{ osp_db_name }}; CREATE USER IF NOT EXISTS {{ osp_db_user}}; GRANT ALL ON DATABASE {{osp_db_name}} TO {{ osp_db_user }};"
+
+- name: Create client certs
+  become: yes
+  command:
+    argv:
+      - cockroach
+      - cert
+      - create-client
+      - "{{ osp_db_user }}"
+      - --certs-dir=/etc/cockroach
+      - --ca-key=/etc/cockroach/ca.key
+    creates: "/etc/cockroach/client.{{ osp_db_user }}.key"
+
+- name: check permission of certs
+  become: yes
+  file:
+    path: "/etc/cockroach/client.{{ osp_db_user }}.{{ item }}"
+    mode: "u=rwx,g=,o="
+    owner: "{{ osp_http_user }}"
+  loop:
+  - key
+  - crt
+
+- name: Set DB location
+  when: osp_db_location != "///db/database.db"
+  set_fact:
+    osp_db_location: "{{ osp_db_location_cockroachdb }}"

tasks/db_mysql.yml

@@ -1,34 +0,0 @@
-- name: DB - Install
-  package:
-    name:
-    - mariadb
-    - python-mysqlclient
-
-- name: DB - Init
-  command: mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
-  args:
-    creates: /var/lib/mysql/
-
-- name: Started
-  systemd:
-    name: mariadb
-    enabled: yes
-    state: started
-
-- name: DB - create user
-  mysql_user:
-    login_host: "{{ osp_db_host }}"
-    name: "{{ osp_db_user }}"
-    password: "{{ osp_db_pass }}"
-    priv: '{{ osp_db_name }}.*:ALL,GRANT'
-
-- name: DB - create database
-  mysql_db:
-    login_host: "{{ osp_db_host }}"
-    name: "{{ osp_db_name }}"
-    encoding: utf8mb4
-
-- name: Set DB location
-  when: osp_db_location != "///db/database.db"
-  set_fact:
-    osp_db_location: "{{ osp_db_location_mysql }}"

tasks/db_postgresql.yml

@@ -1,18 +0,0 @@
-- name: DB - create user
-  postgresql_user:
-    login_host: "{{ osp_db_host }}"
-    name: "{{ osp_db_user }}"
-
-- name: DB - create database
-  postgresql_db:
-    login_host: "{{ osp_db_host }}"
-    name: "{{ osp_db_name }}"
-    owner: "{{ osp_db_user }}"
-    encoding: UTF-8
-    lc_collate: en_US.UTF-8
-    lc_ctype: en_US.UTF-8
-
-- name: Set DB location
-  when: osp_db_location != "///db/database.db"
-  set_fact:
-    osp_db_location: "{{ osp_db_location_postgresql }}"

tasks/main.yml

@@ -17,7 +17,6 @@
     #- python-gevent-websocket
     - base-devel
     - yay
-    - python-requests # ejabberd auth
 
 - name: Create AUR User for build
   user:
@@ -108,10 +107,6 @@
     - live-adapt
     - stream-thumb
 
-- name: tls
-  import_tasks: tls.yml
-  when: osp_tls_enabled
-
 - name: Nginx
   systemd:
     name: nginx
@@ -119,31 +114,26 @@
     enabled: yes
 
 - name: Database
-  include_tasks: db_postgresql.yml
+  include_tasks: db_cockroachdb.yml
-  when: osp_db_type == "postgresql"
+  when: osp_db_type == "cockroachdb"
-
-- name: Database
-  include_tasks: db_mysql.yml
-  when: osp_db_type == "mysql"
 
 - name: Clone OSP repository
   git:
     repo: "{{ osp_git_root }}"
     dest: "/opt/osp/"
     version: "{{ osp_git_commit }}"
-  ignore_errors: yes
 
 - name: Install python requirements
   pip:
     requirements: /opt/osp/setup/requirements.txt
     virtualenv: /opt/osp-venv
 
-#- name: Create cache directory
+- name: Create cache directory
-#  file:
+  file:
-#    path: /var/cache/osp
+    path: /var/cache/osp
-#    owner: "{{ osp_http_user }}"
+    owner: "{{ osp_http_user }}"
-#    group: "{{ osp_http_group }}"
+    group: "{{ osp_http_group }}"
-#    state: directory
+    state: directory
 
 - name: Create logging directory
   file:
@@ -165,19 +155,15 @@
     dest: "/opt/osp/conf/config.py"
     state: link
 
-- name: Configure supply rtmp
+- name: Permissions for database
-  notify: restart osp-rtmp
   file:
-    src: "/etc/osp.conf"
+    path: "/opt/osp/{{ item }}"
-    dest: "/opt/osp/installs/osp-rtmp/conf/config.py"
-    state: link
-
-- name: Create logging directory
-  file:
-    path: /opt/osp
     owner: "{{ osp_http_user }}"
     group: "{{ osp_http_group }}"
     recurse: yes
+  loop:
+    - db
+    - migrations
 
 - name: Init Database
   become: yes
@@ -185,17 +171,7 @@
   command: python3 manage.py db init
   args:
     chdir: /opt/osp/
-    creates: /opt/osp/migrations
+    creates: /opt/osp/db/database.db
-  environment:
-    VIRTUAL_ENV: /opt/osp-venv
-    PATH: /opt/osp-venv/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl
-
-- name: Permissions for database
-  file:
-    path: "/opt/osp/db"
-    owner: "{{ osp_http_user }}"
-    group: "{{ osp_http_group }}"
-    recurse: yes
 
 - name: Install services files and workers
   notify: restart osp
@@ -205,14 +181,9 @@
   loop:
     - osp-worker@.service
     - osp.target
-    - osp-rtmp.service
 
 - name: Start OSP
   systemd:
-    name: "{{ item }}"
+    name: osp.target
     state: started
     enabled: yes
-  loop:
-    - osp.target
-    - osp-rtmp.service
-    - osp.target

tasks/tls.yml

@@ -1,43 +0,0 @@
-- name: Install
-  package:
-    name: dehydrated
-
-- name: create folders
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-  - /etc/nginx/sites.d
-  - /etc/nginx/local.d
-  - /etc/nginx/snippets
-
-- name: create folders
-  file:
-    path: "/srv/http/.well-known/acme-challenge"
-    state: directory
-
-- name: templates
-  notify: reload nginx
-  template:
-    src: "{{ item.file }}"
-    dest: "/etc/nginx/{{ item.path }}"
-  with_items:
-  - file: letsencrypt.nginx
-    path: snippets/letsencrypt.conf
-
-- name: config dehydrated
-  template:
-    src: dehydrated
-    dest: /etc/dehydrated/config
-
-- name: get let's encrypt account 
-  command: /usr/bin/dehydrated --register --accept-terms
-  args:
-    creates: /etc/dehydrated/accounts
-
-- name: get osp_hostname cert
-  notify: dehydrated
-  lineinfile:
-    path: /etc/dehydrated/domains.txt
-    line: "{{ osp_hostname }}"
-    create: yes

templates/config.py.dist

@@ -24,8 +24,6 @@ requireEmailRegistration={{ osp_require_email_registration }}
 # Enables Debug Mode
 debugMode = False
 
-ospCoreAPI = "http://127.0.0.1:{{ osp_worker_start_port }}"
-
 # EJabberD Configuration
 ejabberdAdmin = "admin"
 ejabberdPass = "{{ osp_ejabberd_password }}"

templates/dehydrated

@@ -1,2 +0,0 @@
-WELLKNOWN=/srv/http/.well-known/acme-challenge
-CONTACT_EMAIL={{ webserver_dehydrated_email  }}

templates/ejabberd.yml

@@ -161,7 +161,7 @@ shaper_rules:
 
 auth_use_cache: false
 auth_password_format: scram
-extauth_program: "/usr/bin/python3 /opt/osp/installs/ejabberd/setup/auth_osp.py"
+extauth_program: "/usr/bin/python3 /opt/osp/setup/ejabberd/auth_osp.py"
 extauth_instances: 3
 
 host_config:

templates/letsencrypt.nginx

@@ -1,4 +0,0 @@
-location /.well-known/acme-challenge {
-	alias /srv/http/.well-known/acme-challenge;
-	allow all;
-}

templates/nginx.conf

@@ -17,7 +17,7 @@ http {
     include       mime.types;
     default_type  application/octet-stream;
 
-    proxy_cache_path    /tmp/osp levels=1:2 keys_zone=auth_cache:5m max_size=1g inactive=24h;
+    proxy_cache_path    /var/cache/osp levels=1:2 keys_zone=auth_cache:5m max_size=1g inactive=24h;
 
     sendfile        on;
     tcp_nopush      on;
@@ -65,7 +65,6 @@ http {
     upstream socket_nodes {
 	# sticky only on commercial nginx
         # sticky cookie srv_id expires=8h;
-        hash $remote_addr consistent;
 {% for n in range(osp_worker_count) %}
         server 127.0.0.1:{{ osp_worker_start_port + n }};
 {% endfor %}
@@ -90,32 +89,11 @@ http {
             root /opt/osp/static;
         }
     }
-{% if osp_tls_enabled %}
-    server {
-        listen [::]:80;
-        listen 80;
-        location / {
-            return 301 https://$host$request_uri;
-        }
-
-        include snippets/letsencrypt.conf;
-    }
-{% endif %}
 
     # NGINX to OSP Gunicorn Processes Reverse Proxy
     server {
-{% if osp_tls_enabled %}
-	listen [::]:443 ssl http2;
-	listen 443 ssl http2;
-
-	server_name {{ osp_hostname }};
-
-	ssl_certificate /etc/dehydrated/certs/{{ osp_hostname }}/fullchain.pem;
-	ssl_certificate_key /etc/dehydrated/certs/{{ osp_hostname }}/privkey.pem;
-{% else %}
         listen       80;
         listen       [::]:80;
-{% endif %}
 
         # set client body size to 16M #
         client_max_body_size 16M;
@@ -129,9 +107,7 @@ http {
             proxy_set_header   X-Forwarded-For      $proxy_add_x_forwarded_for;
             proxy_set_header   X-Forwarded-Proto    $scheme;
         }
-{% if osp_tls_enabled %}
+
-	include snippets/letsencrypt.conf;
-{% endif %}
         include osp-socketio.conf;
         include osp-redirects.conf;
 

templates/osp-redirects.conf

@@ -106,12 +106,12 @@
 
         location /edge {
             auth_request /ospAuth;
-            rewrite ^/edge/(.*)$ $scheme://$ospedge_node/edge/$1 redirect;
+            rewrite ^/edge/(.*)$ $scheme://$ospedge_node/live/$1 redirect;
         }
 
         location /edge-adapt {
             auth_request /ospAuth;
-            rewrite ^/edge-adapt/(.*)$ $scheme://$ospedge_node/edge-adapt/$1 redirect;
+            rewrite ^/edge-adapt/(.*)$ $scheme://$ospedge_node/live-adapt/$1 redirect;
         }
 
         location /http-bind/ {              # BOSH XMPP-HTTP

templates/osp-rtmp.conf

@@ -6,9 +6,6 @@ rtmp {
                 #listen 1935;
                 listen [::]:1935;
                 chunk_size 4096;
-                ping 30s;
-                ping_timeout 15s;
-                drop_idle_publisher 15s;
 
                 application stream {
                         live on;
@@ -16,12 +13,10 @@ rtmp {
 
                         allow publish all;
                         #deny publish all;
-                        allow play ::1;
                         allow play 127.0.0.1;
-                        deny play all;
 
-                        on_publish http://127.0.0.1:{{ osp_worker_rtmp_port }}/auth-key;
+                        on_publish http://127.0.0.1:{{ osp_worker_start_port }}/auth-key;
-                        on_publish_done http://127.0.0.1:{{ osp_worker_rtmp_port }}/deauth-user;
+                        on_publish_done http://127.0.0.1:{{ osp_worker_start_port }}/deauth-user;
 
                 }
                 application stream-data {
@@ -29,22 +24,19 @@ rtmp {
 
                         allow publish all;
                         #deny publish all;
-                        allow play ::1;
                         allow play 127.0.0.1;
-                        deny play all;
 
-                        on_publish http://127.0.0.1:{{ osp_worker_rtmp_port }}/auth-user;
+                        on_publish http://127.0.0.1:{{ osp_worker_start_port }}/auth-user;
                         push rtmp://127.0.0.1:1935/live/;
                         push rtmp://127.0.0.1:1935/record/;
 
                         hls on;
                         hls_path {{ osp_http_path }}/live;
                         hls_fragment 1;
-                        #hls_playlist_length 30s;
+                        hls_playlist_length 30s;
-                        hls_playlist_length 30m;
 
                         hls_nested on;
-                        hls_fragment_naming sequential;
+                        hls_fragment_naming system;
 
                         recorder thumbnail {
                             record video;
@@ -62,19 +54,16 @@ rtmp {
 
                         allow publish all;
                         #deny publish all;
-                        allow play ::1;
                         allow play 127.0.0.1;
-                        deny play all;
 
-                        on_publish http://127.0.0.1:{{ osp_worker_rtmp_port }}/auth-user;
+                        on_publish http://127.0.0.1:{{ osp_worker_start_port }}/auth-user;
                         push rtmp://127.0.0.1:1935/live/;
                         push rtmp://127.0.0.1:1935/record/;
 
                         exec ffmpeg -i rtmp://127.0.0.1:1935/live/$name
-                                # -c:v libx264 -c:a aac -b:a 192k -vf "scale=-2:1080" -vsync 1 -copyts -start_at_zero -sws_flags lanczos -r 30 -g 30 -keyint_min 30 -force_key_frames "expr:gte(t,n_forced*1)" -tune zerolatency -preset ultrafast -crf 28 -maxrate 4192k -bufsize 8384k -threads 4 -f flv rtmp://127.0.0.1:1935/show/$name_1080
+                                -c:v libx264 -c:a aac -b:a 128k -vf "scale=-2:720" -vsync 1 -copyts -start_at_zero -sws_flags lanczos -r 30 -g 30 -keyint_min 30 -force_key_frames "expr:gte(t,n_forced*1)" -tune zerolatency -preset ultrafast -crf 28 -maxrate 2096k -bufsize 4192k -threads 16 -f flv rtmp://localhost:1935/show/$name_720
-                                -c:v libx264 -c:a aac -b:a 128k -vf "scale=-2:720" -vsync 1 -copyts -start_at_zero -sws_flags lanczos -r 30 -g 30 -keyint_min 30 -force_key_frames "expr:gte(t,n_forced*1)" -tune zerolatency -preset ultrafast -crf 28 -maxrate 2096k -bufsize 4192k -threads 4 -f flv rtmp://127.0.0.1:1935/show/$name_720
+                                -c:v libx264 -c:a aac -b:a 96k -vf "scale=-2:480" -vsync 1 -copyts -start_at_zero -sws_flags lanczos -r 30 -g 30 -keyint_min 30 -force_key_frames "expr:gte(t,n_forced*1)" -tune zerolatency -preset ultrafast -crf 28 -maxrate 1200k -bufsize 2400k -threads 16 -f flv rtmp://localhost:1935/show/$name_480
-                                -c:v libx264 -c:a aac -b:a 96k -vf "scale=-2:480" -vsync 1 -copyts -start_at_zero -sws_flags lanczos -r 30 -g 30 -keyint_min 30 -force_key_frames "expr:gte(t,n_forced*1)" -tune zerolatency -preset ultrafast -crf 28 -maxrate 1200k -bufsize 2400k -threads 4 -f flv rtmp://127.0.0.1:1935/show/$name_480;
+                                -c copy -f flv rtmp://localhost:1935/show/$name_src;
-                                # -c copy -f flv rtmp://127.0.0.1:1935/show/$name_src;
 
 
                         recorder thumbnail {
@@ -91,42 +80,32 @@ rtmp {
 
                 application show {
                         live on;
-                        allow publish ::1;
                         allow publish 127.0.0.1;
-                        deny publish all;
-                        allow play ::1;
                         allow play 127.0.0.1;
-                        deny play all;
 
                         hls on;
                         hls_path {{ osp_http_path }}/live-adapt;
                         hls_nested on;
                         hls_fragment 1;
-                        #hls_playlist_length 30s;
+                        hls_playlist_length 30s;
-                        hls_playlist_length 30m;
 
-                        hls_fragment_naming sequential;
+                        hls_fragment_naming system;
 
                         record off;
 
                         # Instruct clients to adjust resolution according to bandwidth
                         hls_variant _480 BANDWIDTH=1200000; # Medium bitrate, SD resolution
                         hls_variant _720 BANDWIDTH=2048000; # High bitrate, HD 720p resolution
-                        hls_variant _1080 BANDWIDTH=4096000; # FHB 1080p
+                        hls_variant _src BANDWIDTH=4096000; # Source bitrate, source resolution
-                        #hls_variant _src BANDWIDTH=4096000; # Source bitrate, source resolution
                 }
 
                 application record {
                         live on;
 
-                        allow publish ::1;
                         allow publish 127.0.0.1;
-                        deny publish all;
-                        allow play ::1;
                         allow play 127.0.0.1;
-                        deny play all;
 
-                        on_publish http://127.0.0.1:{{ osp_worker_rtmp_port }}/auth-record;
+                        on_publish http://127.0.0.1:{{ osp_worker_start_port }}/auth-record;
                         exec_push mkdir -m 764 {{ osp_http_path }}/videos/$name;
 
                         recorder all {
@@ -137,18 +116,17 @@ rtmp {
                             exec_record_done bash -c "ffmpeg -y -i $path -codec copy -movflags +faststart {{ osp_http_path }}/videos/$name/$basename.mp4 && rm $path";
                             exec_record_done mv {{ osp_http_path }}/stream-thumb/$name.png {{ osp_http_path }}/videos/$name/$basename.png;
                             exec_record_done mv {{ osp_http_path }}/stream-thumb/$name.gif {{ osp_http_path }}/videos/$name/$basename.gif;
-                            on_record_done http://127.0.0.1:{{ osp_worker_rtmp_port }}/deauth-record;
+                            on_record_done http://127.0.0.1:{{ osp_worker_start_port }}/deauth-record;
                         }
                 }
 
                 application live {
                         live on;
-                        allow publish ::1;
+                        drop_idle_publisher 30s;
                         allow publish 127.0.0.1;
-                        deny publish all;
                         allow play all;
 
-                        on_play http://127.0.0.1:{{ osp_worker_rtmp_port }}/playbackAuth;
+                        on_play http://127.0.0.1:{{ osp_worker_start_port }}/playbackAuth;
                 }
         }
 }

templates/osp-rtmp.service

@@ -1,24 +0,0 @@
-[Unit]
-Description=Gunicorn instance to serve OSP-RTMP
-After=network.target ejabberd.service redis.service
-PartOf=osp.target
-
-{% if osp_db_type == "postgresql" %}
-After=postgresql.service
-{% endif %}
-{% if osp_db_type == "mysql" %}
-After=mysql.service
-{% endif %}
-
-[Service]
-User=http
-Group=http
-WorkingDirectory=/opt/osp/installs/osp-rtmp/
-Environment="VIRTUAL_ENV=/opt/osp-venv"
-Environment="PATH=/opt/osp-venv/bin:/usr/local/bin:/usr/bin:/bin"
-ExecStart=/opt/osp-venv/bin/gunicorn app:app -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker -w 1 --bind 127.0.0.1:{{ osp_worker_rtmp_port }} --reload --access-logfile /var/log/osp/access.log --error-logfile /var/log/osp/error.log
-Restart=on-failure
-RestartSec=5m
-
-[Install]
-WantedBy=multi-user.target

templates/osp-worker@.service

@@ -3,20 +3,13 @@ Description=Gunicorn instance to serve OSP Workers on port %i
 After=network.target ejabberd.service redis.service
 PartOf=osp.target
 
-{% if osp_db_type == "postgresql" %}
-After=postgresql.service
-{% endif %}
-{% if osp_db_type == "mysql" %}
-After=mysql.service
-{% endif %}
-
 [Service]
 User=http
 Group=http
 WorkingDirectory=/opt/osp
 Environment="VIRTUAL_ENV=/opt/osp-venv"
 Environment="PATH=/opt/osp-venv/bin:/usr/local/bin:/usr/bin:/bin"
-ExecStart=/opt/osp-venv/bin/gunicorn app:app -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker -w {{ osp_worker_worker }} --bind 0.0.0.0:%i  --timeout {{ osp_worker_timeout }} --reload --access-logfile /var/log/osp/access.log --error-logfile /var/log/osp/error.log
+ExecStart=/opt/osp-venv/bin/gunicorn app:app -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker -w 1 --bind 0.0.0.0:%i --reload --access-logfile /var/log/osp/access.log --error-logfile /var/log/osp/error.log
 Restart=on-failure
 RestartSec=5m