From 470032569e821cc8adc0d201ba626e408cc43339 Mon Sep 17 00:00:00 2001 From: Adnan Maolood Date: Mon, 13 Jun 2022 16:52:23 -0400 Subject: [PATCH] buildsrht: Use GraphQL to submit builds Use the GraphQL API to submit builds so that job creation webhooks are delivered. --- buildsrht/blueprints/api.py | 178 +++++++++++++++++++---------------- buildsrht/blueprints/jobs.py | 15 +-- buildsrht/runner.py | 24 ++--- master-shell | 18 +--- 4 files changed, 114 insertions(+), 121 deletions(-) diff --git a/buildsrht/blueprints/api.py b/buildsrht/blueprints/api.py index 561e454..5a23235 100644 --- a/buildsrht/blueprints/api.py +++ b/buildsrht/blueprints/api.py @@ -3,9 +3,10 @@ from srht.api import paginated_response from srht.config import cfg from srht.database import db from srht.flask import csrf_bypass +from srht.graphql import exec_gql from srht.validation import Validation from srht.oauth import oauth, current_token -from buildsrht.runner import queue_build, requires_payment +from buildsrht.runner import requires_payment from buildsrht.types import Artifact, Job, JobStatus, Task, JobGroup from buildsrht.types import Trigger, TriggerType, TriggerCondition from buildsrht.manifest import Manifest @@ -40,53 +41,67 @@ def jobs_POST(): "Manifest must be less than {} bytes".format(max_len), field="manifest") note = valid.optional("note", cls=str) - read = valid.optional("access:read", ["*"], list) - write = valid.optional("access:write", [current_token.user.username], list) - secrets = valid.optional("secrets", cls=bool, default=True) + secrets = valid.optional("secrets", cls=bool) tags = valid.optional("tags", [], list) valid.expect(not valid.ok or all(re.match(r"^[A-Za-z0-9_.-]+$", tag) for tag in tags), "Invalid tag name, tags must use lowercase alphanumeric characters, underscores, dashes, or dots", field="tags") - triggers = valid.optional("triggers", list(), list) - execute = valid.optional("execute", True, bool) + execute = valid.optional("execute", cls=bool) if not valid.ok: return valid.response - try: - manifest = Manifest(yaml.safe_load(_manifest)) - except Exception as ex: - valid.error(str(ex)) + + resp = exec_gql("builds.sr.ht", """ + mutation SubmitBuild( + $manifest: String!, + $note: String, + $tags: [String!], + $secrets: Boolean, + $execute: Boolean, + ) { + submit( + manifest: $manifest, + note: $note, + tags: $tags, + secrets: $secrets, + execute: $execute, + ) { + id + log { + fullURL + } + tasks { + name + status + log { + fullURL + } + } + note + runner + tags + owner { + canonical_name: canonicalName + ... on User { + name: username + } + } + } + } + """, user=current_token.user, valid=valid, manifest=_manifest, note=note, tags=tags, secrets=secrets, execute=execute) + + if not valid.ok: return valid.response - # TODO: access controls - job = Job(current_token.user, _manifest) - job.image = manifest.image - job.note = note - if tags: - job.tags = "/".join(tags) - job.secrets = secrets - db.session.add(job) - db.session.flush() - for task in manifest.tasks: - t = Task(job, task.name) - db.session.add(t) - db.session.flush() # assigns IDs for ordering purposes - for index, trigger in enumerate(triggers): - _valid = Validation(trigger) - action = _valid.require("action", TriggerType) - condition = _valid.require("condition", TriggerCondition) - if not _valid.ok: - _valid.copy(valid, "triggers[{}]".format(index)) - return valid.response - # TODO: Validate details based on trigger type - t = Trigger(job) - t.trigger_type = action - t.condition = condition - t.details = json.dumps(trigger) - db.session.add(t) - if execute: - queue_build(job, manifest) # commits the session - else: - db.session.commit() - return job.to_dict() + + resp = resp["submit"] + if resp["log"]: + resp["setup_log"] = resp["log"]["fullURL"] + del resp["log"] + resp["tags"] = "/".join(resp["tags"]) + for task in resp["tasks"]: + task["status"] = task["status"].lower() + if task["log"]: + task["log"] = task["log"]["fullURL"] + return resp @api.route("/api/jobs/") @oauth("jobs:read") @@ -134,7 +149,11 @@ def jobs_by_id_start_POST(job_id): { "reason": "This job is already {}".format(reason_map.get(job.status)) } ] }, 400 - queue_build(job, Manifest(yaml.safe_load(job.manifest))) + exec_gql("builds.sr.ht", """ + mutation StartJob($jobId: Int!) { + start(jobID: $jobId) { id } + } + """, user=current_token.user, jobId=job.id) return { } @api.route("/api/jobs//cancel", methods=["POST"]) @@ -151,7 +170,6 @@ def jobs_by_id_cancel_POST(job_id): @api.route("/api/job-group", methods=["POST"]) @oauth("jobs:write") def job_group_POST(): - # TODO: implement starting the job group right away valid = Validation(request) jobs = valid.require("jobs") valid.expect(not jobs or isinstance(jobs, list) and all(isinstance(j, int) for j in jobs), @@ -165,49 +183,45 @@ def job_group_POST(): if not valid.ok: return valid.response - job_group = JobGroup() - job_group.note = note - job_group.owner_id = current_token.user_id - db.session.add(job_group) - db.session.flush() - - for job_id in jobs: - job = Job.query.filter(Job.id == job_id).one_or_none() - valid.expect(job, f"Job ID {job_id} not found") - if not job: - continue - valid.expect(job.status == JobStatus.pending, - f"Job ID {job.id} has already been started; submit jobs with execute=false to create groups") - valid.expect(job.owner_id == current_token.user_id, - f"Job ID {job_id} is not owned by you") - valid.expect(not job.job_group_id, - f"Job ID {job_id} is already assigned to a job group") - job.job_group_id = job_group.id - job_group.jobs.append(job) - - for index, trigger in enumerate(triggers): - _valid = Validation(trigger) - action = _valid.require("action", TriggerType) - condition = _valid.require("condition", TriggerCondition) - if not _valid.ok: - _valid.copy(valid, "triggers[{}]".format(index)) - return valid.response - t = Trigger(job_group) - t.trigger_type = action - t.condition = condition - t.details = json.dumps(trigger) - db.session.add(t) + triggers = [{ + "type": trigger["action"].upper(), + "condition": trigger["condition"].upper(), + "email": { + "to": trigger["to"], + "cc": trigger.get("cc"), + "inReplyTo": trigger.get("in_reply_to"), + } if trigger["action"] == "email" else None, + "webhook": { + "url": trigger["url"], + } if trigger["action"] == "webhook" else None, + } for trigger in triggers] + + resp = exec_gql("builds.sr.ht", """ + mutation CreateJobGroup($jobIds: [Int!]!, $triggers: [TriggerInput!], $execute: Boolean, $note: String) { + createGroup(jobIds: $jobIds, triggers: $triggers, execute: $execute, note: $note) { + id + note + owner { + canonical_name: canonicalName + ... on User { + name: username + } + } + jobs { + id + status + } + } + } + """, user=current_token.user, valid=valid, jobIds=jobs, triggers=triggers, execute=execute, note=note) if not valid.ok: return valid.response - db.session.commit() - if execute: - for job in job_group.jobs: - queue_build(job, Manifest(yaml.safe_load(job.manifest))) - db.session.commit() - - return job_group.to_dict() + resp = resp["createGroup"] + for job in resp["jobs"]: + job["status"] = job["status"].lower() + return resp @api.route("/api/job-group/") @oauth("jobs:read") diff --git a/buildsrht/blueprints/jobs.py b/buildsrht/blueprints/jobs.py index 9b47376..67c820f 100644 --- a/buildsrht/blueprints/jobs.py +++ b/buildsrht/blueprints/jobs.py @@ -1,7 +1,7 @@ from ansi2html import Ansi2HTMLConverter from buildsrht.manifest import Manifest from buildsrht.rss import generate_feed -from buildsrht.runner import queue_build, requires_payment +from buildsrht.runner import submit_build, requires_payment from buildsrht.search import apply_search from buildsrht.types import Job, JobStatus, Task, TaskStatus, User from datetime import datetime, timedelta @@ -249,17 +249,8 @@ def submit_POST(): except Exception as ex: valid.error(str(ex), field="manifest") return render_template("submit.html", **valid.kwargs) - job = Job(current_user, _manifest) - job.image = manifest.image - job.note = note - db.session.add(job) - db.session.flush() - for task in manifest.tasks: - t = Task(job, task.name) - db.session.add(t) - db.session.flush() # assigns IDs for ordering purposes - queue_build(job, manifest) # commits the session - return redirect("/~" + current_user.username + "/job/" + str(job.id)) + job_id = submit_build(current_user, _manifest, note=note) + return redirect("/~" + current_user.username + "/job/" + str(job_id)) @jobs.route("/cancel/", methods=["POST"]) @loginrequired diff --git a/buildsrht/runner.py b/buildsrht/runner.py index 7773452..dcecc39 100644 --- a/buildsrht/runner.py +++ b/buildsrht/runner.py @@ -4,6 +4,7 @@ from datetime import datetime from srht.config import cfg from srht.database import db from srht.email import send_email +from srht.graphql import exec_gql from srht.oauth import UserType from srht.metrics import RedisQueueCollector from prometheus_client import Counter @@ -22,20 +23,15 @@ runner = Celery('builds', broker=builds_broker, config_source={ builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue") builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted") -def queue_build(job, manifest): - from buildsrht.types import JobStatus - job.status = JobStatus.queued - db.session.commit() - # crypto mining attempt - # pretend to accept it and let the admins know - sample = json.dumps(manifest.to_dict()) - if "xuirig" in sample or "miner" in sample or "selci" in sample: - send_email(f"User {job.owner.canonical_name} attempted to submit cryptocurrency mining job #{job.id}", - cfg("sr.ht", "owner-email"), - "Cryptocurrency mining attempt on builds.sr.ht") - else: - builds_submitted.inc() - run_build.delay(job.id, manifest.to_dict()) +def submit_build(user, manifest, note=None, tags=[]): + resp = exec_gql("builds.sr.ht", """ + mutation SubmitBuild($manifest: String!, $tags: [String!], $note: String) { + submit(manifest: $manifest, tags: $tags, note: $note) { + id + } + } + """, user=user, manifest=manifest, note=note, tags=tags) + return resp["submit"]["id"] def requires_payment(user): if allow_free: diff --git a/master-shell b/master-shell index 6e1b360..e8bbf2c 100755 --- a/master-shell +++ b/master-shell @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from buildsrht.manifest import Manifest -from buildsrht.runner import queue_build +from buildsrht.runner import submit_build from buildsrht.types import Job, Task, User from getopt import getopt, GetoptError from srht.config import cfg, get_origin @@ -43,18 +43,10 @@ if cmd[0] == "submit": manifest = Manifest(yaml.safe_load(_manifest)) except Exception as ex: fail(str(ex)) - job = Job(user, _manifest) - job.image = manifest.image - job.note = " ".join([y for x, y in opts if x == "-n"]) or None - job.tags = "/".join([y for x, y in opts if x == "-t"]) or None - db.session.add(job) - db.session.flush() - for task in manifest.tasks: - t = Task(job, task.name) - db.session.add(t) - db.session.flush() # assigns IDs for ordering purposes - queue_build(job, manifest) # commits the session - url = f"{get_origin('builds.sr.ht', external=True)}/~{username}/job/{job.id}" + note = " ".join([y for x, y in opts if x == "-n"]) or None + tags = [y for x, y in opts if x == "-t"] or None + job_id = submit_build(user, _manifest, note=note, tags=tags) + url = f"{get_origin('builds.sr.ht', external=True)}/~{username}/job/{job_id}" print(url) else: fail(f"Unknown command {cmd[0]}") -- 2.38.5