From fe9b32680447b6e64779b3f1a692f4ec08ad4007 Mon Sep 17 00:00:00 2001 From: Jakob Meier Date: Tue, 31 Oct 2023 15:18:46 +0100 Subject: [PATCH] experimental git.sr.ht support TODO: documentation --- roles/git.sr.ht/defaults/main.yml | 3 + roles/git.sr.ht/tasks/config.yml | 102 ++++++++++++ roles/git.sr.ht/tasks/db.yml | 15 ++ roles/git.sr.ht/tasks/main.yml | 15 ++ roles/git.sr.ht/tasks/nginx.yml | 13 ++ roles/git.sr.ht/templates/nginx.conf | 50 ++++++ roles/git.sr.ht/templates/schema.psql | 227 ++++++++++++++++++++++++++ run.yml | 8 + 8 files changed, 433 insertions(+) create mode 100644 roles/git.sr.ht/defaults/main.yml create mode 100644 roles/git.sr.ht/tasks/config.yml create mode 100644 roles/git.sr.ht/tasks/db.yml create mode 100644 roles/git.sr.ht/tasks/main.yml create mode 100644 roles/git.sr.ht/tasks/nginx.yml create mode 100644 roles/git.sr.ht/templates/nginx.conf create mode 100644 roles/git.sr.ht/templates/schema.psql diff --git a/roles/git.sr.ht/defaults/main.yml b/roles/git.sr.ht/defaults/main.yml new file mode 100644 index 0000000..ab3b20d --- /dev/null +++ b/roles/git.sr.ht/defaults/main.yml @@ -0,0 +1,3 @@ +--- +gitsrht_oauth_client_id: "" +gitsrht_oauth_client_secret: "" diff --git a/roles/git.sr.ht/tasks/config.yml b/roles/git.sr.ht/tasks/config.yml new file mode 100644 index 0000000..95d6011 --- /dev/null +++ b/roles/git.sr.ht/tasks/config.yml @@ -0,0 +1,102 @@ +--- +- name: Ensure the git.sr.ht config is injected + ansible.builtin.blockinfile: + path: /etc/sr.ht/config.ini + marker: "#-- {mark} ANSIBLE git.sr.ht --#" + block: | + + [git.sr.ht] + # + # URL git.sr.ht is being served at (protocol://domain) + origin={{ srht_protocol }}://git.{{ srht_domain }} + # + # Address and port to bind the debug server to + debug-host=0.0.0.0 + debug-port=5001 + # + # Configures the SQLAlchemy connection string for the database. + connection-string=postgresql://postgres@127.0.0.1/gitsrht?sslmode=disable + # + # Set to "yes" to automatically run migrations on package upgrade. + migrate-on-upgrade=yes + # + # The redis connection used for the webhooks worker + webhooks=redis://127.0.0.1:6379/1 + # + # A post-update script which is installed in every git repo. + post-update-script=/usr/bin/gitsrht-update-hook + # + # git.sr.ht's OAuth client ID and secret for meta.sr.ht + # Register your client at meta.example.org/oauth + oauth-client-id={{ gitsrht_oauth_client_id }} + oauth-client-secret={{ gitsrht_oauth_client_secret }} + # + # Path to git repositories on disk + repos=/var/lib/git/ + # + # Configure the S3 bucket and prefix for object storage. Leave empty to disable + # object storage. Bucket is required to enable object storage; prefix is + # optional. + s3-bucket= + s3-prefix= + # + # Required for preparing and sending patchsets from git.sr.ht + outgoing-domain= + + # + # Origin URL for the API + # Only needed if not run behind a reverse proxy, e.g. for local development. + # By default, the API port is 100 more than the web port + #api-origin=http://localhost:5101 + + [git.sr.ht::api] + # + # Maximum complexity of GraphQL queries. The higher this number, the more work + # that API clients can burden the API backend with. Complexity is equal to the + # number of discrete fields which would be returned to the user. 200 is a good + # default. + max-complexity=200 + + # + # The maximum time the API backend will spend processing a single API request. + # + # See https://golang.org/pkg/time/#ParseDuration + max-duration=3s + + # + # Set of IP subnets which are permitted to utilize internal API + # authentication. This should be limited to the subnets from which your + # *.sr.ht services are running. + # + # Comma-separated, CIDR notation. + internal-ipnet=127.0.0.0/8,::1/128,192.168.0.0/16,10.0.0.0/8 + [git.sr.ht::dispatch] + # + # The authorized keys hook uses this to dispatch to various handlers + # The format is a program to exec into as the key, and the user to match as the + # value. When someone tries to log in as this user, this program is executed + # and is expected to omit an AuthorizedKeys file. + # + # Uncomment the relevant lines to enable the various sr.ht dispatchers. + /usr/bin/gitsrht-keys=git:git + /usr/bin/buildsrht-keys=builds:builds + register: conf + +- name: Enable & start git.sr.ht service + ansible.builtin.service: + name: git.sr.ht + state: restarted + enabled: true + when: conf.changed +- name: Enable & start git.sr.ht api service + ansible.builtin.service: + name: git.sr.ht-api + state: restarted + enabled: true + when: conf.changed +- name: Enable & start git.sr.ht webhooks service + ansible.builtin.service: + name: git.sr.ht-webhooks + state: restarted + enabled: true + when: conf.changed diff --git a/roles/git.sr.ht/tasks/db.yml b/roles/git.sr.ht/tasks/db.yml new file mode 100644 index 0000000..9a0c585 --- /dev/null +++ b/roles/git.sr.ht/tasks/db.yml @@ -0,0 +1,15 @@ +--- +- name: Copy database schema to host + ansible.builtin.template: + src: schema.psql + dest: /tmp/gitsrht.psql + +- name: Create database + community.postgresql.postgresql_db: + name: gitsrht + +- name: Ensure database layout + community.postgresql.postgresql_db: + name: gitsrht + state: restore + target: /tmp/gitsrht.psql diff --git a/roles/git.sr.ht/tasks/main.yml b/roles/git.sr.ht/tasks/main.yml new file mode 100644 index 0000000..91bafef --- /dev/null +++ b/roles/git.sr.ht/tasks/main.yml @@ -0,0 +1,15 @@ +--- +- name: Install git.sr.ht packages + community.general.apk: + name: + - git.sr.ht + state: latest + +- name: Setup Database + ansible.builtin.import_tasks: db.yml + +- name: Setup config & services + ansible.builtin.import_tasks: config.yml + +- name: Setup nginx + ansible.builtin.import_tasks: nginx.yml diff --git a/roles/git.sr.ht/tasks/nginx.yml b/roles/git.sr.ht/tasks/nginx.yml new file mode 100644 index 0000000..bffcb78 --- /dev/null +++ b/roles/git.sr.ht/tasks/nginx.yml @@ -0,0 +1,13 @@ +--- +- name: Copy nginx config file + ansible.builtin.template: + src: nginx.conf + dest: /etc/nginx/http.d/git.sr.ht.conf + register: nginxconf + +- name: Start & enable nginx + ansible.builtin.service: + name: nginx + state: restarted + enabled: true + when: nginxconf.changed diff --git a/roles/git.sr.ht/templates/nginx.conf b/roles/git.sr.ht/templates/nginx.conf new file mode 100644 index 0000000..f665043 --- /dev/null +++ b/roles/git.sr.ht/templates/nginx.conf @@ -0,0 +1,50 @@ +server { + include sourcehut.conf; + server_name git.{{ srht_domain }}; + + client_max_body_size 100M; + + location ^~ /.well-known { + root /var/www; + } + + location = /robots.txt { + root /var/www; + } + + location / { + proxy_pass http://127.0.0.1:5001; + include headers.conf; + add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src * data:; script-src 'self' 'unsafe-inline'; frame-ancestors 'none'" always; + include web.conf; + } + + location /query { + proxy_pass http://127.0.0.1:5101; + include graphql.conf; + } + + location /static { + root /usr/lib/$python/site-packages/gitsrht; + expires 30d; + } + + location = /authorize { + proxy_pass http://127.0.0.1:5001; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + } + + location ~ ^/([^/]+)/([^/]+)/(HEAD|info/refs|objects/info/.*|git-upload-pack).*$ { + auth_request /authorize; + root /var/lib/git; + fastcgi_pass unix:/run/fcgiwrap/fcgiwrap.sock; + fastcgi_param SCRIPT_FILENAME /usr/libexec/git-core/git-http-backend; + fastcgi_param PATH_INFO $uri; + fastcgi_param GIT_PROJECT_ROOT $document_root; + fastcgi_read_timeout 500s; + include fastcgi_params; + gzip off; + } +} diff --git a/roles/git.sr.ht/templates/schema.psql b/roles/git.sr.ht/templates/schema.psql new file mode 100644 index 0000000..45a60b0 --- /dev/null +++ b/roles/git.sr.ht/templates/schema.psql @@ -0,0 +1,227 @@ +CREATE TYPE auth_method AS ENUM ( + 'OAUTH_LEGACY', + 'OAUTH2', + 'COOKIE', + 'INTERNAL', + 'WEBHOOK' +); + +CREATE TYPE clone_status AS ENUM ( + 'NONE', + 'IN_PROGRESS', + 'COMPLETE', + 'ERROR' +); + +CREATE TYPE visibility AS ENUM ( + 'PUBLIC', + 'PRIVATE', + 'UNLISTED' +); + +CREATE TYPE webhook_event AS ENUM ( + 'REPO_CREATED', + 'REPO_UPDATE', + 'REPO_DELETED' +); + +CREATE TYPE owner_repo_name AS ( + owner text, + repo_name text +); + +CREATE TYPE owner_id_repo_name AS ( + owner_id integer, + repo_name text +); + +CREATE TABLE "user" ( + id serial PRIMARY KEY, + username character varying(256) UNIQUE, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + email character varying(256) NOT NULL, + user_type character varying NOT NULL DEFAULT 'active_non_paying'::character varying, + url character varying(256), + location character varying(256), + bio character varying(4096), + suspension_notice character varying(4096), + -- TODO: Delete these + oauth_token character varying(256), + oauth_token_expires timestamp without time zone, + oauth_token_scopes character varying DEFAULT ''::character varying, + oauth_revocation_token character varying(256) +); + +CREATE INDEX ix_user_username ON "user" USING btree (username); + +CREATE TABLE repository ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + name character varying(256) NOT NULL, + description character varying(1024), + owner_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + path character varying(1024), + visibility visibility NOT NULL, + readme character varying, + clone_status clone_status NOT NULL, + clone_error character varying, + CONSTRAINT repository_check + CHECK (((clone_status = 'ERROR'::clone_status) <> (clone_error IS NULL))), + CONSTRAINT uq_repo_owner_id_name UNIQUE (owner_id, name) +); + +CREATE TABLE access ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + repo_id integer NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + mode character varying NOT NULL, + CONSTRAINT uq_access_user_id_repo_id UNIQUE (user_id, repo_id) +); + +CREATE TABLE artifacts ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + user_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + repo_id integer NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + commit character varying NOT NULL, + filename character varying NOT NULL, + checksum character varying NOT NULL, + size integer NOT NULL, + CONSTRAINT repo_artifact_filename_unique UNIQUE (repo_id, filename) +); + +CREATE TABLE redirect ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + name character varying(256) NOT NULL, + owner_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + path character varying(1024), + new_repo_id integer NOT NULL REFERENCES repository(id) ON DELETE CASCADE +); + +-- GraphQL webhooks +CREATE TABLE gql_user_wh_sub ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + events webhook_event[] NOT NULL, + url character varying NOT NULL, + query character varying NOT NULL, + auth_method auth_method NOT NULL, + token_hash character varying(128), + grants character varying, + client_id uuid, + expires timestamp without time zone, + node_id character varying, + user_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + CONSTRAINT gql_user_wh_sub_auth_method_check + CHECK ((auth_method = ANY (ARRAY['OAUTH2'::auth_method, 'INTERNAL'::auth_method]))), + CONSTRAINT gql_user_wh_sub_check + CHECK (((auth_method = 'OAUTH2'::auth_method) = (token_hash IS NOT NULL))), + CONSTRAINT gql_user_wh_sub_check1 + CHECK (((auth_method = 'OAUTH2'::auth_method) = (expires IS NOT NULL))), + CONSTRAINT gql_user_wh_sub_check2 + CHECK (((auth_method = 'INTERNAL'::auth_method) = (node_id IS NOT NULL))), + CONSTRAINT gql_user_wh_sub_events_check + CHECK ((array_length(events, 1) > 0)) +); + +CREATE INDEX gql_user_wh_sub_token_hash_idx + ON gql_user_wh_sub + USING btree (token_hash); + +CREATE TABLE gql_user_wh_delivery ( + id serial PRIMARY KEY, + uuid uuid NOT NULL, + date timestamp without time zone NOT NULL, + event webhook_event NOT NULL, + subscription_id integer NOT NULL + REFERENCES gql_user_wh_sub(id) ON DELETE CASCADE, + request_body character varying NOT NULL, + response_body character varying, + response_headers character varying, + response_status integer +); + +-- Legacy SSH key table, to be fetched from meta.sr.ht instead (TODO: Remove) +CREATE TABLE sshkey ( + id serial PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + meta_id integer NOT NULL, + key character varying(4096) NOT NULL, + fingerprint character varying(512) NOT NULL +); + +CREATE INDEX ix_sshkey_key + ON sshkey + USING btree (md5((key)::text)); + +CREATE UNIQUE INDEX ix_sshkey_meta_id + ON sshkey + USING btree (meta_id); + +-- Legacy OAuth (TODO: Remove) +CREATE TABLE oauthtoken ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + updated timestamp without time zone NOT NULL, + expires timestamp without time zone NOT NULL, + user_id integer REFERENCES "user"(id) ON DELETE CASCADE, + token_hash character varying(128) NOT NULL, + token_partial character varying(8) NOT NULL, + scopes character varying(512) NOT NULL +); + +-- Legacy webhooks (TODO: Remove) +CREATE TABLE user_webhook_subscription ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + url character varying(2048) NOT NULL, + events character varying NOT NULL, + user_id integer REFERENCES "user"(id) ON DELETE CASCADE, + token_id integer REFERENCES oauthtoken(id) ON DELETE CASCADE +); + +CREATE TABLE user_webhook_delivery ( + id serial PRIMARY KEY, + uuid uuid NOT NULL, + created timestamp without time zone NOT NULL, + event character varying(256) NOT NULL, + url character varying(2048) NOT NULL, + payload character varying(65536) NOT NULL, + payload_headers character varying(16384) NOT NULL, + response character varying(65536), + response_status integer NOT NULL, + response_headers character varying(16384), + subscription_id integer NOT NULL + REFERENCES user_webhook_subscription(id) ON DELETE CASCADE +); + +CREATE TABLE repo_webhook_subscription ( + id serial PRIMARY KEY, + created timestamp without time zone NOT NULL, + url character varying(2048) NOT NULL, + events character varying NOT NULL, + user_id integer REFERENCES "user"(id) ON DELETE CASCADE, + token_id integer REFERENCES oauthtoken(id) ON DELETE CASCADE, + repo_id integer REFERENCES repository(id) ON DELETE CASCADE, + sync boolean DEFAULT false NOT NULL +); + +CREATE TABLE repo_webhook_delivery ( + id serial PRIMARY KEY, + uuid uuid NOT NULL, + created timestamp without time zone NOT NULL, + event character varying(256) NOT NULL, + url character varying(2048) NOT NULL, + payload character varying(65536) NOT NULL, + payload_headers character varying(16384) NOT NULL, + response character varying(65536), + response_status integer NOT NULL, + response_headers character varying(16384), + subscription_id integer NOT NULL + REFERENCES repo_webhook_subscription(id) ON DELETE CASCADE +); diff --git a/run.yml b/run.yml index ce8df9f..472f624 100644 --- a/run.yml +++ b/run.yml @@ -11,3 +11,11 @@ hosts: all roles: - role: meta.sr.ht + +############################################ +# OPTIONAL SOURCEHUT MODULES +############################################ +- name: Setup git.sr.ht + hosts: all + roles: + - role: git.sr.ht -- 2.38.5