From a32ed87a84e8c5a57c9d3759440a5f500e612dfc Mon Sep 17 00:00:00 2001 From: Jakob Meier Date: Tue, 31 Oct 2023 15:19:50 +0100 Subject: [PATCH] experimental builds.sr.ht support TODO: documentation TODO: proper docker setup --- roles/builds.sr.ht/defaults/main.yml | 3 + roles/builds.sr.ht/tasks/config.yml | 106 ++++++++++++++ roles/builds.sr.ht/tasks/db.yml | 15 ++ roles/builds.sr.ht/tasks/main.yml | 17 +++ roles/builds.sr.ht/tasks/nginx.yml | 13 ++ roles/builds.sr.ht/templates/nginx.conf | 23 ++++ roles/builds.sr.ht/templates/schema.psql | 168 +++++++++++++++++++++++ run.yml | 4 + 8 files changed, 349 insertions(+) create mode 100644 roles/builds.sr.ht/defaults/main.yml create mode 100644 roles/builds.sr.ht/tasks/config.yml create mode 100644 roles/builds.sr.ht/tasks/db.yml create mode 100644 roles/builds.sr.ht/tasks/main.yml create mode 100644 roles/builds.sr.ht/tasks/nginx.yml create mode 100644 roles/builds.sr.ht/templates/nginx.conf create mode 100644 roles/builds.sr.ht/templates/schema.psql diff --git a/roles/builds.sr.ht/defaults/main.yml b/roles/builds.sr.ht/defaults/main.yml new file mode 100644 index 0000000..2e6c347 --- /dev/null +++ b/roles/builds.sr.ht/defaults/main.yml @@ -0,0 +1,3 @@ +--- +buildssrht_oauth_client_id: "" +buildssrht_oauth_client_secret: "" diff --git a/roles/builds.sr.ht/tasks/config.yml b/roles/builds.sr.ht/tasks/config.yml new file mode 100644 index 0000000..b8fc81c --- /dev/null +++ b/roles/builds.sr.ht/tasks/config.yml @@ -0,0 +1,106 @@ +--- +- name: Ensure the builds.sr.ht config is injected + ansible.builtin.blockinfile: + path: /etc/sr.ht/config.ini + marker: "#-- {mark} ANSIBLE builds.sr.ht --#" + block: | + [builds.sr.ht] + # + # URL builds.sr.ht is being served at (protocol://domain) + origin={{ srht_protocol }}://builds.{{ srht_domain }} + # + # Address and port to bind the debug server to + debug-host=0.0.0.0 + debug-port=5002 + # + # Configures the SQLAlchemy connection string for the database. + connection-string=postgresql://postgres@127.0.0.1/buildsrht?sslmode=disable + # + # Set to "yes" to automatically run migrations on package upgrade. + migrate-on-upgrade=yes + # + # The redis connection used for the Celery worker (configure this on both the + # master and workers) + redis=redis://127.0.0.1:6379/0 + # + # builds.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + oauth-client-id={{ buildssrht_oauth_client_id }} + oauth-client-secret={{ buildssrht_oauth_client_id }} + # + # Script used to launch on ssh connection. /usr/bin/master-shell on master, + # /usr/bin/runner-shell for workers. + # If master and worker are on the same system set to /usr/bin/runner-shell + shell=/usr/bin/runner-shell + # + # Set to "yes" to allow nonpaying users to submit builds + allow-free=yes + # + # Origin URL for the API + # Only needed if not run behind a reverse proxy, e.g. for local development. + # By default, the API port is 100 more than the web port + # api-origin=http://127.0.0.1:5102 + + # + # These config options are only necessary for systems running a build runner + [builds.sr.ht::worker] + # + # Name of this build runner (with HTTP port if not 80) + name=runner.{{ srht_domain }} + # + # Path to write build logs + buildlogs=./logs + # + # Path to the build images + images=./images + # + # In production you should NOT put the build user in the docker group. Instead, + # make a scratch user who is and write a sudoers or doas.conf file that allows + # them to execute just the control command, then update this config option. For + # example: + # + # doas -u docker /var/lib/images/control + # + # Assuming doas.conf looks something like this: + # + # permit nopass builds as docker cmd /var/lib/images/control + # + # For more information about the security model of builds.sr.ht, visit the wiki: + # + # https://man.sr.ht/builds.sr.ht/installation.md + controlcmd=./images/control + # + # Max build duration. See https://golang.org/pkg/time/#ParseDuration + timeout=45m + # + # Http bind address for serving local build information/monitoring + bind-address=0.0.0.0:8080 + # + # Build trigger email + trigger-from={{ srht_smtp_from }} + # + # Configure the S3 bucket and prefix for object storage. Leave empty to disable + # object storage. Bucket is required to enable object storage; prefix is + # optional. + s3-bucket= + s3-prefix= + register: conf + +- name: Enable & start builds.sr.ht service + ansible.builtin.service: + name: builds.sr.ht + state: restarted + enabled: true + when: conf.changed +- name: Enable & start builds.sr.ht api service + ansible.builtin.service: + name: builds.sr.ht-api + state: restarted + enabled: true + when: conf.changed +- name: Enable & start builds.sr.ht worker service + ansible.builtin.service: + name: builds.sr.ht-worker + state: restarted + enabled: true + when: conf.changed diff --git a/roles/builds.sr.ht/tasks/db.yml b/roles/builds.sr.ht/tasks/db.yml new file mode 100644 index 0000000..4c049b5 --- /dev/null +++ b/roles/builds.sr.ht/tasks/db.yml @@ -0,0 +1,15 @@ +--- +- name: Copy database schema to host + ansible.builtin.template: + src: schema.psql + dest: /tmp/buildssrht.psql + +- name: Create database + community.postgresql.postgresql_db: + name: buildssrht + +- name: Ensure database layout + community.postgresql.postgresql_db: + name: buildssrht + state: restore + target: /tmp/buildssrht.psql diff --git a/roles/builds.sr.ht/tasks/main.yml b/roles/builds.sr.ht/tasks/main.yml new file mode 100644 index 0000000..5f48597 --- /dev/null +++ b/roles/builds.sr.ht/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Install builds.sr.ht packages + community.general.apk: + name: + - builds.sr.ht + - builds.sr.ht-images + - builds.sr.ht-worker + state: latest + +- name: Setup Database + ansible.builtin.import_tasks: db.yml + +- name: Setup config & services + ansible.builtin.import_tasks: config.yml + +- name: Setup nginx + ansible.builtin.import_tasks: nginx.yml diff --git a/roles/builds.sr.ht/tasks/nginx.yml b/roles/builds.sr.ht/tasks/nginx.yml new file mode 100644 index 0000000..2c42dde --- /dev/null +++ b/roles/builds.sr.ht/tasks/nginx.yml @@ -0,0 +1,13 @@ +--- +- name: Copy nginx config file + ansible.builtin.template: + src: nginx.conf + dest: /etc/nginx/http.d/hub.sr.ht.conf + register: nginxconf + +- name: Start & enable nginx + ansible.builtin.service: + name: nginx + state: restarted + enabled: true + when: nginxconf.changed diff --git a/roles/builds.sr.ht/templates/nginx.conf b/roles/builds.sr.ht/templates/nginx.conf new file mode 100644 index 0000000..8e848d0 --- /dev/null +++ b/roles/builds.sr.ht/templates/nginx.conf @@ -0,0 +1,23 @@ +server { + include sourcehut.conf; + server_name hub.{{ srht_domain }}; + + client_max_body_size 100M; + + location / { + proxy_pass http://127.0.0.1:5002; + include headers.conf; + add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; script-src 'self' 'unsafe-inline'; frame-ancestors 'none'" always; + include web.conf; + } + + location /query { + proxy_pass http://127.0.0.1:5102; + include graphql.conf; + } + + location /static { + root /usr/lib/$python/site-packages/buildsrht; + expires 30d; + } +} diff --git a/roles/builds.sr.ht/templates/schema.psql b/roles/builds.sr.ht/templates/schema.psql new file mode 100644 index 0000000..cb53ea7 --- /dev/null +++ b/roles/builds.sr.ht/templates/schema.psql @@ -0,0 +1,168 @@ +CREATE TYPE auth_method AS ENUM ( + 'OAUTH_LEGACY', + 'OAUTH2', + 'COOKIE', + 'INTERNAL', + 'WEBHOOK' +); + +CREATE TYPE webhook_event AS ENUM ( + 'JOB_CREATED' +); + +CREATE TYPE visibility AS ENUM ( + 'PUBLIC', + 'UNLISTED', + 'PRIVATE' +); + +CREATE TABLE "user" ( + id serial PRIMARY KEY, + username character varying(256) UNIQUE, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + oauth_token character varying(256), + oauth_token_expires timestamp without time zone, + email character varying(256) NOT NULL, + user_type character varying DEFAULT 'active_non_paying'::character varying NOT NULL, + url character varying(256), + location character varying(256), + bio character varying(4096), + oauth_token_scopes character varying DEFAULT 'profile'::character varying, + oauth_revocation_token character varying(256), + suspension_notice character varying(4096) +); + +CREATE TABLE secret ( + id serial PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + uuid uuid NOT NULL, + name character varying(512), + from_user_id integer REFERENCES "user"(id) ON DELETE SET NULL, + -- Key secrets: + secret_type character varying NOT NULL, + secret bytea NOT NULL, + -- File secrets: + path character varying(512), + mode integer, + CONSTRAINT secret_user_id_uuid_unique UNIQUE (user_id, uuid) +); + +CREATE INDEX ix_user_username ON "user" USING btree (username); + +CREATE TABLE job_group ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + owner_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + note character varying(4096) +); + +CREATE TABLE job ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + manifest character varying(16384) NOT NULL, + owner_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + job_group_id integer REFERENCES job_group(id) ON DELETE SET NULL, + note character varying(4096), + tags character varying, + runner character varying, + status character varying NOT NULL, + secrets boolean DEFAULT true NOT NULL, + image character varying(128), + visibility visibility NOT NULL +); + +CREATE INDEX ix_job_owner_id ON job USING btree (owner_id); + +CREATE INDEX ix_job_job_group_id ON job USING btree (job_group_id); + +CREATE TABLE artifact ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + job_id integer NOT NULL REFERENCES job(id) ON DELETE CASCADE, + name character varying NOT NULL, + path character varying NOT NULL, + url character varying NOT NULL, + size integer NOT NULL +); + +CREATE TABLE task ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + name character varying(256) NOT NULL, + status character varying NOT NULL, + job_id integer NOT NULL REFERENCES job(id) ON DELETE CASCADE +); + +CREATE INDEX ix_task_job_id ON task USING btree (job_id); + +CREATE TABLE trigger ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + details character varying(4096) NOT NULL, + condition character varying NOT NULL, + trigger_type character varying NOT NULL, + job_id integer REFERENCES job(id) ON DELETE CASCADE, + job_group_id integer REFERENCES job_group(id) ON DELETE CASCADE +); + +-- GraphQL webhooks +CREATE TABLE gql_user_wh_sub ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + events webhook_event[] NOT NULL, + url character varying NOT NULL, + query character varying NOT NULL, + auth_method auth_method NOT NULL, + token_hash character varying(128), + grants character varying, + client_id uuid, + expires timestamp without time zone, + node_id character varying, + user_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + CONSTRAINT gql_user_wh_sub_auth_method_check + CHECK ((auth_method = ANY (ARRAY['OAUTH2'::auth_method, 'INTERNAL'::public.auth_method]))), + CONSTRAINT gql_user_wh_sub_check + CHECK (((auth_method = 'OAUTH2'::auth_method) = (token_hash IS NOT NULL))), + CONSTRAINT gql_user_wh_sub_check1 + CHECK (((auth_method = 'OAUTH2'::auth_method) = (expires IS NOT NULL))), + CONSTRAINT gql_user_wh_sub_check2 + CHECK (((auth_method = 'INTERNAL'::auth_method) = (node_id IS NOT NULL))), + CONSTRAINT gql_user_wh_sub_events_check + CHECK ((array_length(events, 1) > 0)) +); + +CREATE INDEX gql_user_wh_sub_token_hash_idx + ON gql_user_wh_sub + USING btree (token_hash); + +CREATE TABLE gql_user_wh_delivery ( + id serial PRIMARY KEY, + uuid uuid NOT NULL, + date timestamp without time zone NOT NULL, + event webhook_event NOT NULL, + subscription_id integer NOT NULL + REFERENCES gql_user_wh_sub(id) ON DELETE CASCADE, + request_body character varying NOT NULL, + response_body character varying, + response_headers character varying, + response_status integer +); + +-- Legacy OAuth +CREATE TABLE oauthtoken ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + expires timestamp without time zone NOT NULL, + user_id integer REFERENCES "user"(id) ON DELETE CASCADE, + token_hash character varying(128) NOT NULL, + token_partial character varying(8) NOT NULL, + scopes character varying(512) NOT NULL +); diff --git a/run.yml b/run.yml index f11ce56..39c1239 100644 --- a/run.yml +++ b/run.yml @@ -23,3 +23,7 @@ hosts: all roles: - role: hub.sr.ht +- name: Setup builds.sr.ht + hosts: all + roles: + - role: builds.sr.ht -- 2.38.5