~comcloudway/builds.sr.ht

bc87a4ba1a08e7f37988f048e991444a5242dac6 — Adnan Maolood 1 year, 6 months ago 89c87b2
Implement job visibility

This implements visibility for build jobs. The visibility can be set
when submitting a build, and can also be changed retroactively from a
new job settings page.
M api/graph/model/job.go => api/graph/model/job.go +9 -7
@@ 15,13 15,14 @@ import (
)

type Job struct {
	ID       int       `json:"id"`
	Created  time.Time `json:"created"`
	Updated  time.Time `json:"updated"`
	Manifest string    `json:"manifest"`
	Note     *string   `json:"note"`
	Image    string    `json:"image"`
	Runner   *string   `json:"runner"`
	ID         int        `json:"id"`
	Created    time.Time  `json:"created"`
	Updated    time.Time  `json:"updated"`
	Manifest   string     `json:"manifest"`
	Note       *string    `json:"note"`
	Image      string     `json:"image"`
	Runner     *string    `json:"runner"`
	Visibility Visibility `json:"visibility"`

	OwnerID    int
	JobGroupID *int


@@ 75,6 76,7 @@ func (j *Job) Fields() *database.ModelFields {
			{"tags", "tags", &j.RawTags},
			{"status", "status", &j.RawStatus},
			{"image", "image", &j.Image},
			{"visibility", "visibility", &j.Visibility},

			// Always fetch:
			{"id", "", &j.ID},

M api/graph/schema.graphqls => api/graph/schema.graphqls +8 -1
@@ 103,6 103,12 @@ enum JobStatus {
  CANCELLED
}

enum Visibility {
  PUBLIC
  UNLISTED
  PRIVATE
}

type Job {
  id: Int!
  created: Time!


@@ 111,6 117,7 @@ type Job {
  manifest: String!
  note: String
  tags: [String!]!
  visibility: Visibility!

  "Name of the build image"
  image: String!


@@ 437,7 444,7 @@ type Mutation {
  executed immediately if unspecified.
  """
  submit(manifest: String!, tags: [String!] note: String, secrets: Boolean,
    execute: Boolean): Job! @access(scope: JOBS, kind: RW)
    execute: Boolean, visibility: Visibility): Job! @access(scope: JOBS, kind: RW)

  "Queues a pending job."
  start(jobID: Int!): Job @access(scope: JOBS, kind: RW)

M api/graph/schema.resolvers.go => api/graph/schema.resolvers.go +27 -8
@@ 189,6 189,7 @@ func (r *jobGroupResolver) Owner(ctx context.Context, obj *model.JobGroup) (mode

// Jobs is the resolver for the jobs field.
func (r *jobGroupResolver) Jobs(ctx context.Context, obj *model.JobGroup) ([]*model.Job, error) {
	user := auth.ForContext(ctx)
	var jobs []*model.Job
	if err := database.WithTx(ctx, &sql.TxOptions{
		Isolation: 0,


@@ 198,7 199,13 @@ func (r *jobGroupResolver) Jobs(ctx context.Context, obj *model.JobGroup) ([]*mo
		rows, err := database.
			Select(ctx, job).
			From(`job j`).
			Where(`j.job_group_id = ?`, obj.ID).
			Where(sq.And{
				sq.Expr(`j.job_group_id = ?`, obj.ID),
				sq.Or{
					sq.Expr(`j.owner_id = ?`, user.UserID),
					sq.Expr(`j.visibility = 'PUBLIC'`),
				},
			}).
			RunWith(tx).
			QueryContext(ctx)
		if err != nil {


@@ 256,7 263,7 @@ func (r *jobGroupResolver) Triggers(ctx context.Context, obj *model.JobGroup) ([
}

// Submit is the resolver for the submit field.
func (r *mutationResolver) Submit(ctx context.Context, manifest string, tags []string, note *string, secrets *bool, execute *bool) (*model.Job, error) {
func (r *mutationResolver) Submit(ctx context.Context, manifest string, tags []string, note *string, secrets *bool, execute *bool, visibility *model.Visibility) (*model.Job, error) {
	man, err := LoadManifest(manifest)
	if err != nil {
		return nil, err


@@ 264,6 271,11 @@ func (r *mutationResolver) Submit(ctx context.Context, manifest string, tags []s
	conf := config.ForContext(ctx)
	user := auth.ForContext(ctx)

	vis := model.VisibilityUnlisted
	if visibility != nil {
		vis = *visibility
	}

	allowFree, _ := conf.Get("builds.sr.ht", "allow-free")
	if allowFree != "yes" {
		if user.UserType != "admin" &&


@@ 288,19 300,19 @@ func (r *mutationResolver) Submit(ctx context.Context, manifest string, tags []s
		// TODO: Refactor tags into a pg array
		row := tx.QueryRowContext(ctx, `INSERT INTO job (
			created, updated,
			manifest, owner_id, secrets, note, tags, image, status
			manifest, owner_id, secrets, note, tags, image, status, visibility
		) VALUES (
			NOW() at time zone 'utc',
			NOW() at time zone 'utc',
			$1, $2, $3, $4, $5, $6, $7
			$1, $2, $3, $4, $5, $6, $7, $8
		) RETURNING
			id, created, updated, manifest, note, image, runner, owner_id,
			tags, status
		`, manifest, user.UserID, sec, note, tags, man.Image, status)
			tags, status, visibility
		`, manifest, user.UserID, sec, note, tags, man.Image, status, vis)

		if err := row.Scan(&job.ID, &job.Created, &job.Updated, &job.Manifest,
			&job.Note, &job.Image, &job.Runner, &job.OwnerID, &job.RawTags,
			&job.RawStatus); err != nil {
			&job.RawStatus, &job.Visibility); err != nil {
			return err
		}



@@ 928,6 940,7 @@ func (r *userResolver) Jobs(ctx context.Context, obj *model.User, cursor *coremo
		cursor = coremodel.NewCursor(nil)
	}

	user := auth.ForContext(ctx)
	var jobs []*model.Job
	if err := database.WithTx(ctx, &sql.TxOptions{
		Isolation: 0,


@@ 937,7 950,13 @@ func (r *userResolver) Jobs(ctx context.Context, obj *model.User, cursor *coremo
		query := database.
			Select(ctx, job).
			From(`job j`).
			Where(`j.owner_id = ?`, obj.ID)
			Where(sq.And{
				sq.Expr(`j.owner_id = ?`, obj.ID),
				sq.Or{
					sq.Expr(`j.owner_id = ?`, user.UserID),
					sq.Expr(`j.visibility = 'PUBLIC'`),
				},
			})
		jobs, cursor = job.QueryWithCursor(ctx, tx, query, cursor)
		return nil
	}); err != nil {

M api/loaders/middleware.go => api/loaders/middleware.go +9 -1
@@ 11,6 11,7 @@ import (
	"github.com/lib/pq"

	"git.sr.ht/~sircmpwn/builds.sr.ht/api/graph/model"
	"git.sr.ht/~sircmpwn/core-go/auth"
	"git.sr.ht/~sircmpwn/core-go/database"
)



@@ 118,6 119,7 @@ func fetchUsersByName(ctx context.Context) func(names []string) ([]*model.User, 
}

func fetchJobsByID(ctx context.Context) func(ids []int) ([]*model.Job, []error) {
	user := auth.ForContext(ctx)
	return func(ids []int) ([]*model.Job, []error) {
		jobs := make([]*model.Job, len(ids))
		if err := database.WithTx(ctx, &sql.TxOptions{


@@ 131,7 133,13 @@ func fetchJobsByID(ctx context.Context) func(ids []int) ([]*model.Job, []error) 
			query := database.
				Select(ctx, (&model.Job{}).As("job")).
				From(`job`).
				Where(sq.Expr(`job.id = ANY(?)`, pq.Array(ids)))
				Where(sq.And{
					sq.Expr(`job.id = ANY(?)`, pq.Array(ids)),
					sq.Or{
						sq.Expr(`job.owner_id = ?`, user.UserID),
						sq.Expr(`job.visibility != 'PRIVATE'`),
					},
				})
			if rows, err = query.RunWith(tx).QueryContext(ctx); err != nil {
				return err
			}

A buildsrht/alembic/versions/ae3544d6450a_add_visibility_to_job.py => buildsrht/alembic/versions/ae3544d6450a_add_visibility_to_job.py +41 -0
@@ 0,0 1,41 @@
"""Add visibility to job

Revision ID: ae3544d6450a
Revises: 76bb268d91f7
Create Date: 2023-03-13 10:33:49.830104

"""

# revision identifiers, used by Alembic.
revision = 'ae3544d6450a'
down_revision = '76bb268d91f7'

from alembic import op
import sqlalchemy as sa


def upgrade():
    op.execute("""
    CREATE TYPE visibility AS ENUM (
        'PUBLIC',
        'UNLISTED',
        'PRIVATE'
    );

    ALTER TABLE job
    ADD COLUMN visibility visibility;

    UPDATE job
    SET visibility = 'UNLISTED'::visibility;

    ALTER TABLE job
    ALTER COLUMN visibility
    SET NOT NULL;
    """)


def downgrade():
    op.execute("""
    ALTER TABLE job DROP COLUMN visibility;
    DROP TYPE visibility;
    """)

M buildsrht/app.py => buildsrht/app.py +2 -0
@@ 28,10 28,12 @@ class BuildApp(SrhtFlask):
        from buildsrht.blueprints.api import api
        from buildsrht.blueprints.jobs import jobs
        from buildsrht.blueprints.secrets import secrets
        from buildsrht.blueprints.settings import settings
        from srht.graphql import gql_blueprint

        self.register_blueprint(admin)
        self.register_blueprint(api)
        self.register_blueprint(settings)
        self.register_blueprint(jobs)
        self.register_blueprint(secrets)
        self.register_blueprint(gql_blueprint)

M buildsrht/blueprints/jobs.py => buildsrht/blueprints/jobs.py +29 -8
@@ 3,7 3,7 @@ from buildsrht.manifest import Manifest
from buildsrht.rss import generate_feed
from buildsrht.runner import submit_build, requires_payment
from buildsrht.search import apply_search
from buildsrht.types import Job, JobStatus, Task, TaskStatus, User
from buildsrht.types import Job, JobStatus, Task, TaskStatus, User, Visibility
from datetime import datetime, timedelta
from flask import Blueprint, render_template, request, abort, redirect
from flask import Response, url_for


@@ 35,6 35,23 @@ metrics = type("metrics", tuple(), {

requests_session = requests.Session()

def get_access(job, user=None):
    user = user or current_user

    # Anonymous
    if not user:
        if job.visibility == Visibility.PRIVATE:
            return False
        return True

    # Owner
    if user.id == job.owner_id:
        return True

    if job.visibility == Visibility.PRIVATE:
        return False
    return True

def tags(tags):
    if not tags:
        return list()


@@ 239,6 256,7 @@ def submit_POST():
    valid.expect(not _manifest or len(_manifest) < max_len,
            "Manifest must be less than {} bytes".format(max_len),
            field="manifest")
    visibility = valid.require("visibility")
    payment_required = requires_payment(current_user)
    valid.expect(not payment_required,
            "A paid account is required to submit new jobs")


@@ 249,7 267,8 @@ def submit_POST():
    except Exception as ex:
        valid.error(str(ex), field="manifest")
        return render_template("submit.html", **valid.kwargs)
    job_id = submit_build(current_user, _manifest, note=note)
    job_id = submit_build(current_user, _manifest, note=note,
            visibility=visibility)
    return redirect("/~" + current_user.username + "/job/" + str(job_id))

@jobs.route("/cancel/<int:job_id>", methods=["POST"])


@@ 269,8 288,8 @@ def user(username):
    if not user:
        abort(404)
    jobs = Job.query.filter(Job.owner_id == user.id)
    if not current_user or current_user.id != user.id:
        pass # TODO: access controls
    if not current_user or user.id != current_user.id:
        jobs = jobs.filter(Job.visibility == Visibility.PUBLIC)
    origin = cfg("builds.sr.ht", "origin")
    rss_feed = {
        "title": f"{user.username}'s jobs",


@@ 287,8 306,8 @@ def user_rss(username):
    if not user:
        abort(404)
    jobs = Job.query.filter(Job.owner_id == user.id)
    if not current_user or current_user.id != user.id:
        pass  # TODO: access controls
    if not current_user or user.id != current_user.id:
        jobs = jobs.filter(Job.visibility == Visibility.PUBLIC)
    return jobs_feed(jobs, f"{user.username}'s jobs",
                     "jobs.user", username=username)



@@ 316,7 335,7 @@ def tag(username, path):
    jobs = Job.query.filter(Job.owner_id == user.id)\
        .filter(Job.tags.ilike(path + "%"))
    if not current_user or current_user.id != user.id:
        pass # TODO: access controls
        jobs = jobs.filter(Job.visibility == Visibility.PUBLIC)
    origin = cfg("builds.sr.ht", "origin")
    rss_feed = {
        "title": "/".join([f"~{user.username}"] +


@@ 336,7 355,7 @@ def tag_rss(username, path):
    jobs = Job.query.filter(Job.owner_id == user.id)\
        .filter(Job.tags.ilike(path + "%"))
    if not current_user or current_user.id != user.id:
        pass  # TODO: access controls
        jobs = jobs.filter(Job.visibility == Visibility.PUBLIC)
    base_title = "/".join([f"~{user.username}"] +
                          [t["name"] for t in tags(path)])
    return jobs_feed(jobs, base_title + " jobs",


@@ 406,6 425,8 @@ def job_by_id(username, job_id):
    job = Job.query.options(sa.orm.joinedload(Job.tasks)).get(job_id)
    if not job:
        abort(404)
    if not get_access(job):
        abort(404)
    logs = list()
    build_user = cfg("git.sr.ht::dispatch", "/usr/bin/buildsrht-keys", "builds:builds").split(":")[0]
    final_status = [

A buildsrht/blueprints/settings.py => buildsrht/blueprints/settings.py +42 -0
@@ 0,0 1,42 @@
from flask import Blueprint, current_app, render_template, request, url_for, abort, redirect
from flask import current_app
from srht.database import db
from srht.oauth import current_user, loginrequired
from srht.validation import Validation
from buildsrht.types import Job, Visibility

settings = Blueprint("settings", __name__)

@settings.route("/~<username>/job/<int:job_id>/settings/details")
@loginrequired
def details_GET(username, job_id):
    job = Job.query.get(job_id)
    if not job:
        abort(404)
    if current_user.id != job.owner_id:
        abort(404)
    return render_template("job-details.html",
        view="details", job=job)

@settings.route("/~<username>/job/<int:job_id>/settings/details", methods=["POST"])
@loginrequired
def details_POST(username, job_id):
    job = Job.query.get(job_id)
    if not job:
        abort(404)
    if current_user.id != job.owner_id:
        abort(404)

    valid = Validation(request)
    visibility = valid.require("visibility")
    if not valid.ok:
        return render_template("job-details.html",
            job=job, **valid.kwargs), 400

    # TODO: GraphQL mutation to update job details
    job.visibility = visibility
    db.session.commit()

    return redirect(url_for("settings.details_GET",
        username=job.owner.username,
        job_id=job.id))

M buildsrht/runner.py => buildsrht/runner.py +4 -4
@@ 22,14 22,14 @@ runner = Celery('builds', broker=builds_broker, config_source={
builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue")
builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted")

def submit_build(user, manifest, note=None, tags=[]):
def submit_build(user, manifest, note=None, tags=[], visibility=None):
    resp = exec_gql("builds.sr.ht", """
        mutation SubmitBuild($manifest: String!, $tags: [String!], $note: String) {
            submit(manifest: $manifest, tags: $tags, note: $note) {
        mutation SubmitBuild($manifest: String!, $tags: [String!], $note: String, $visibility: Visibility) {
            submit(manifest: $manifest, tags: $tags, note: $note, visibility: $visibility) {
                id
            }
        }
    """, user=user, manifest=manifest, note=note, tags=tags)
    """, user=user, manifest=manifest, note=note, tags=tags, visibility=visibility)
    return resp["submit"]["id"]

def requires_payment(user):

A buildsrht/templates/job-details.html => buildsrht/templates/job-details.html +75 -0
@@ 0,0 1,75 @@
{% extends "settings.html" %}
{% block title %}
<title>Configure {{url_for("jobs.user", username=job.owner.username)}}/#{{job.id}}
  &mdash; {{ cfg("sr.ht", "site-name") }}</title>
{% endblock %}
{% block content %}
<form class="row" method="POST">
  {{csrf_token()}}
  <div class="col-md-6 d-flex flex-column">
    <fieldset class="form-group">
      <legend>Job Visibility</legend>
      <div class="form-check">
        <label class="form-check-label">
          <input
            class="form-check-input"
            type="radio"
            name="visibility"
            value="PUBLIC"
            {% if job.visibility.value == "PUBLIC" %}
            checked
            {% endif %}
            > Public
          <small id="visibility-public-help" class="form-text text-muted">
            Shown on your profile page
          </small>
        </label>
      </div>
      <div class="form-check">
        <label
            class="form-check-label"
            title="Visible to anyone with the link, but not shown on your profile"
          >
          <input
            class="form-check-input"
            type="radio"
            name="visibility"
            value="UNLISTED"
            {% if job.visibility.value == "UNLISTED" %}
            checked
            {% endif %}
            > Unlisted
          <small id="visibility-unlisted-help" class="form-text text-muted">
            Visible to anyone who knows the URL, but not shown on your profile
          </small>
        </label>
      </div>
      <div class="form-check">
        <label
          class="form-check-label"
          title="Only visible to you and your collaborators"
        >
          <input
            class="form-check-input"
            type="radio"
            name="visibility"
            value="PRIVATE"
            {% if job.visibility.value == "PRIVATE" %}
            checked
            {% endif %}
            > Private
          <small id="visibility-unlisted-help" class="form-text text-muted">
            Only visible to you and your collaborators
          </small>
        </label>
      </div>
    </fieldset>
    {{ valid.summary() }}
    <span class="pull-right">
      <button type="submit" class="btn btn-primary">
        Save {{icon("caret-right")}}
      </button>
    </span>
  </div>
</form>
{% endblock %}

M buildsrht/templates/job.html => buildsrht/templates/job.html +36 -5
@@ 13,15 13,46 @@
{% endif %}
{% endblock %}
{% block body %} 
<div class="header-tabbed">
  <div class="container-fluid">
    <h2>
      <a href="{{ url_for("jobs.user", username=job.owner.username) }}">{{ job.owner }}</a>/<wbr
     >#{{ job.id }}
    </h2>
    <ul class="nav nav-tabs">
      {% if job.visibility.value != "PUBLIC" %}
      <li
        class="nav-item nav-text vis-{{job.visibility.value.lower()}}"
        {% if job.visibility.value == "UNLISTED" %}
        title="This job is only visible to those who know the URL."
        {% elif job.visibility.value == "PRIVATE" %}
        title="This job is only visible to those who were invited to view it."
        {% endif %}
      >
        {% if job.visibility.value == "UNLISTED" %}
        Unlisted
        {% elif job.visibility.value == "PRIVATE" %}
        Private
        {% endif %}
      </li>
      {% endif %}
      {% if current_user and current_user.id == job.owner_id %}
      <li class="nav-item">
        <a class="nav-link" href="{{url_for("settings.details_GET",
          username=job.owner.username,
          job_id=job.id)}}"
        >settings</a>
      </li>
      {% endif %}
    </ul>
  </div>
</div>
<div class="container-fluid">
  <section class="row">
    <div class="col-lg-3 col-md-12">
      <h2>
        #{{ job.id }}
        <span class="pull-right">
          {{icon(icon_map.get(job.status), cls=status_map.get(job.status, ""))}}
          {{ job.status.value }}
        </span>
        {{icon(icon_map.get(job.status), cls=status_map.get(job.status, ""))}}
        {{ job.status.value }}
      </h2>
      <dl>
        {% if job.note %}

A buildsrht/templates/settings.html => buildsrht/templates/settings.html +31 -0
@@ 0,0 1,31 @@
{% extends "layout.html" %}
{% block body %}
<div class="header-tabbed">
  <div class="container">
    {% macro link(path, title) %}
    <a
      class="nav-link {% if view == title %}active{% endif %}"
      href="{{ path }}">{{ title }}</a>
    {% endmacro %}
    <h2>
      <a href="{{ url_for("jobs.user", username=job.owner.username) }}">{{ job.owner }}</a>/<wbr
     >#{{ job.id }}
    </h2>
    <ul class="nav nav-tabs">
      <li class="nav-item">
        <a class="nav-link"
         href="{{ url_for("jobs.job_by_id", username=job.owner.username, job_id=job.id) }}"
        >{{icon("caret-left")}}&nbsp;back</a>
      </li>
      <li class="nav-item">
        {{link(url_for("settings.details_GET",
          username=job.owner.username,
          job_id=job.id), "details")}}
      </li>
    </ul>
  </div>
</div>
<div class="container">
  {% block content %}{% endblock %}
</div>
{% endblock %}

M buildsrht/templates/submit.html => buildsrht/templates/submit.html +40 -0
@@ 70,6 70,46 @@
        rows="{{note_rows}}"
      >{{note if note else ""}}</textarea>
    </div>
    <fieldset class="form-group">
      <legend>Visibility</legend>
      <div class="form-check form-check-inline">
        <label
          class="form-check-label"
          title="Publically visible and listed on your profile"
        >
          <input
            class="form-check-input"
            type="radio"
            name="visibility"
            value="PUBLIC"> Public
        </label>
      </div>
      <div class="form-check form-check-inline">
        <label
            class="form-check-label"
            title="Visible to anyone with the link, but not shown on your profile"
          >
          <input
            class="form-check-input"
            type="radio"
            name="visibility"
            value="UNLISTED"
            checked> Unlisted
        </label>
      </div>
      <div class="form-check form-check-inline">
        <label
          class="form-check-label"
          title="Only visible to you and your collaborators"
        >
          <input
            class="form-check-input"
            type="radio"
            name="visibility"
            value="PRIVATE"> Private
        </label>
      </div>
    </fieldset>
    <div class="form-group">
      <a
        class="pull-right"

M buildsrht/types/__init__.py => buildsrht/types/__init__.py +1 -1
@@ 7,7 7,7 @@ class User(Base, ExternalUserMixin):
class OAuthToken(Base, ExternalOAuthTokenMixin):
    pass

from .job import Job, JobStatus
from .job import Job, JobStatus, Visibility
from .task import Task, TaskStatus
from .job_group import JobGroup
from .trigger import Trigger, TriggerType, TriggerCondition

M buildsrht/types/job.py => buildsrht/types/job.py +6 -0
@@ 13,6 13,11 @@ class JobStatus(Enum):
    timeout = 'timeout'
    cancelled = 'cancelled'

class Visibility(Enum):
    PUBLIC = 'PUBLIC'
    UNLISTED = 'UNLISTED'
    PRIVATE = 'PRIVATE'

class Job(Base):
    __tablename__ = 'job'
    id = sa.Column(sa.Integer, primary_key=True)


@@ 32,6 37,7 @@ class Job(Base):
            nullable=False,
            default=JobStatus.pending)
    image = sa.Column(sa.String(256))
    visibility = sa.Column(sau.ChoiceType(Visibility), nullable=False)

    def __init__(self, owner, manifest):
        self.owner_id = owner.id

M schema.sql => schema.sql +8 -1
@@ 10,6 10,12 @@ CREATE TYPE webhook_event AS ENUM (
	'JOB_CREATED'
);

CREATE TYPE visibility AS ENUM (
	'PUBLIC',
	'UNLISTED',
	'PRIVATE'
);

CREATE TABLE "user" (
	id serial PRIMARY KEY,
	username character varying(256) UNIQUE,


@@ 64,7 70,8 @@ CREATE TABLE job (
	runner character varying,
	status character varying NOT NULL,
	secrets boolean DEFAULT true NOT NULL,
	image character varying(128)
	image character varying(128),
	visibility visibility NOT NULL
);

CREATE INDEX ix_job_owner_id ON job USING btree (owner_id);