4 files changed, 114 insertions(+), 121 deletions(-)

M buildsrht/blueprints/api.py
M buildsrht/blueprints/jobs.py
M buildsrht/runner.py
M master-shell

@@ 3,9 3,10 @@ from srht.api import paginated_response
 from srht.config import cfg
 from srht.database import db
 from srht.flask import csrf_bypass
+from srht.graphql import exec_gql
 from srht.validation import Validation
 from srht.oauth import oauth, current_token
-from buildsrht.runner import queue_build, requires_payment
+from buildsrht.runner import requires_payment
 from buildsrht.types import Artifact, Job, JobStatus, Task, JobGroup
 from buildsrht.types import Trigger, TriggerType, TriggerCondition
 from buildsrht.manifest import Manifest


@@ 40,53 41,67 @@ def jobs_POST():
             "Manifest must be less than {} bytes".format(max_len),
             field="manifest")
     note = valid.optional("note", cls=str)
-    read = valid.optional("access:read", ["*"], list)
-    write = valid.optional("access:write", [current_token.user.username], list)
-    secrets = valid.optional("secrets", cls=bool, default=True)
+    secrets = valid.optional("secrets", cls=bool)
     tags = valid.optional("tags", [], list)
     valid.expect(not valid.ok or all(re.match(r"^[A-Za-z0-9_.-]+$", tag) for tag in tags),
         "Invalid tag name, tags must use lowercase alphanumeric characters, underscores, dashes, or dots",
         field="tags")
-    triggers = valid.optional("triggers", list(), list)
-    execute = valid.optional("execute", True, bool)
+    execute = valid.optional("execute", cls=bool)
     if not valid.ok:
         return valid.response
-    try:
-        manifest = Manifest(yaml.safe_load(_manifest))
-    except Exception as ex:
-        valid.error(str(ex))
+
+    resp = exec_gql("builds.sr.ht", """
+        mutation SubmitBuild(
+            $manifest: String!,
+            $note: String,
+            $tags: [String!],
+            $secrets: Boolean,
+            $execute: Boolean,
+        ) {
+            submit(
+                manifest: $manifest,
+                note: $note,
+                tags: $tags,
+                secrets: $secrets,
+                execute: $execute,
+            ) {
+                id
+                log {
+                    fullURL
+                }
+                tasks {
+                    name
+                    status
+                    log {
+                        fullURL
+                    }
+                }
+                note
+                runner
+                tags
+                owner {
+                    canonical_name: canonicalName
+                    ... on User {
+                        name: username
+                    }
+                }
+            }
+        }
+    """, user=current_token.user, valid=valid, manifest=_manifest, note=note, tags=tags, secrets=secrets, execute=execute)
+
+    if not valid.ok:
         return valid.response
-    # TODO: access controls
-    job = Job(current_token.user, _manifest)
-    job.image = manifest.image
-    job.note = note
-    if tags:
-        job.tags = "/".join(tags)
-    job.secrets = secrets
-    db.session.add(job)
-    db.session.flush()
-    for task in manifest.tasks:
-        t = Task(job, task.name)
-        db.session.add(t)
-        db.session.flush() # assigns IDs for ordering purposes
-    for index, trigger in enumerate(triggers):
-        _valid = Validation(trigger)
-        action = _valid.require("action", TriggerType)
-        condition = _valid.require("condition", TriggerCondition)
-        if not _valid.ok:
-            _valid.copy(valid, "triggers[{}]".format(index))
-            return valid.response
-        # TODO: Validate details based on trigger type
-        t = Trigger(job)
-        t.trigger_type = action
-        t.condition = condition
-        t.details = json.dumps(trigger)
-        db.session.add(t)
-    if execute:
-        queue_build(job, manifest) # commits the session
-    else:
-        db.session.commit()
-    return job.to_dict()
+
+    resp = resp["submit"]
+    if resp["log"]:
+        resp["setup_log"] = resp["log"]["fullURL"]
+    del resp["log"]
+    resp["tags"] = "/".join(resp["tags"])
+    for task in resp["tasks"]:
+        task["status"] = task["status"].lower()
+        if task["log"]:
+            task["log"] = task["log"]["fullURL"]
+    return resp
 
 @api.route("/api/jobs/<int:job_id>")
 @oauth("jobs:read")


@@ 134,7 149,11 @@ def jobs_by_id_start_POST(job_id):
                 { "reason": "This job is already {}".format(reason_map.get(job.status)) }
             ]
         }, 400
-    queue_build(job, Manifest(yaml.safe_load(job.manifest)))
+    exec_gql("builds.sr.ht", """
+        mutation StartJob($jobId: Int!) {
+            start(jobID: $jobId) { id }
+        }
+    """, user=current_token.user, jobId=job.id)
     return { }
 
 @api.route("/api/jobs/<int:job_id>/cancel", methods=["POST"])


@@ 151,7 170,6 @@ def jobs_by_id_cancel_POST(job_id):
 @api.route("/api/job-group", methods=["POST"])
 @oauth("jobs:write")
 def job_group_POST():
-    # TODO: implement starting the job group right away
     valid = Validation(request)
     jobs = valid.require("jobs")
     valid.expect(not jobs or isinstance(jobs, list) and all(isinstance(j, int) for j in jobs),


@@ 165,49 183,45 @@ def job_group_POST():
     if not valid.ok:
         return valid.response
 
-    job_group = JobGroup()
-    job_group.note = note
-    job_group.owner_id = current_token.user_id
-    db.session.add(job_group)
-    db.session.flush()
-
-    for job_id in jobs:
-        job = Job.query.filter(Job.id == job_id).one_or_none()
-        valid.expect(job, f"Job ID {job_id} not found")
-        if not job:
-            continue
-        valid.expect(job.status == JobStatus.pending,
-                f"Job ID {job.id} has already been started; submit jobs with execute=false to create groups")
-        valid.expect(job.owner_id == current_token.user_id,
-                f"Job ID {job_id} is not owned by you")
-        valid.expect(not job.job_group_id,
-                f"Job ID {job_id} is already assigned to a job group")
-        job.job_group_id = job_group.id
-        job_group.jobs.append(job)
-
-    for index, trigger in enumerate(triggers):
-        _valid = Validation(trigger)
-        action = _valid.require("action", TriggerType)
-        condition = _valid.require("condition", TriggerCondition)
-        if not _valid.ok:
-            _valid.copy(valid, "triggers[{}]".format(index))
-            return valid.response
-        t = Trigger(job_group)
-        t.trigger_type = action
-        t.condition = condition
-        t.details = json.dumps(trigger)
-        db.session.add(t)
+    triggers = [{
+        "type": trigger["action"].upper(),
+        "condition": trigger["condition"].upper(),
+        "email": {
+            "to": trigger["to"],
+            "cc": trigger.get("cc"),
+            "inReplyTo": trigger.get("in_reply_to"),
+        } if trigger["action"] == "email" else None,
+        "webhook": {
+            "url": trigger["url"],
+        } if trigger["action"] == "webhook" else None,
+    } for trigger in triggers]
+
+    resp = exec_gql("builds.sr.ht", """
+        mutation CreateJobGroup($jobIds: [Int!]!, $triggers: [TriggerInput!], $execute: Boolean, $note: String) {
+            createGroup(jobIds: $jobIds, triggers: $triggers, execute: $execute, note: $note) {
+                id
+                note
+                owner {
+                    canonical_name: canonicalName
+                    ... on User {
+                        name: username
+                    }
+                }
+                jobs {
+                    id
+                    status
+                }
+            }
+        }
+    """, user=current_token.user, valid=valid, jobIds=jobs, triggers=triggers, execute=execute, note=note)
 
     if not valid.ok:
         return valid.response
 
-    db.session.commit()
-    if execute:
-        for job in job_group.jobs:
-            queue_build(job, Manifest(yaml.safe_load(job.manifest)))
-        db.session.commit()
-
-    return job_group.to_dict()
+    resp = resp["createGroup"]
+    for job in resp["jobs"]:
+        job["status"] = job["status"].lower()
+    return resp
 
 @api.route("/api/job-group/<int:job_group_id>")
 @oauth("jobs:read")

@@ 1,7 1,7 @@
 from ansi2html import Ansi2HTMLConverter
 from buildsrht.manifest import Manifest
 from buildsrht.rss import generate_feed
-from buildsrht.runner import queue_build, requires_payment
+from buildsrht.runner import submit_build, requires_payment
 from buildsrht.search import apply_search
 from buildsrht.types import Job, JobStatus, Task, TaskStatus, User
 from datetime import datetime, timedelta


@@ 249,17 249,8 @@ def submit_POST():
     except Exception as ex:
         valid.error(str(ex), field="manifest")
         return render_template("submit.html", **valid.kwargs)
-    job = Job(current_user, _manifest)
-    job.image = manifest.image
-    job.note = note
-    db.session.add(job)
-    db.session.flush()
-    for task in manifest.tasks:
-        t = Task(job, task.name)
-        db.session.add(t)
-        db.session.flush() # assigns IDs for ordering purposes
-    queue_build(job, manifest) # commits the session
-    return redirect("/~" + current_user.username + "/job/" + str(job.id))
+    job_id = submit_build(current_user, _manifest, note=note)
+    return redirect("/~" + current_user.username + "/job/" + str(job_id))
 
 @jobs.route("/cancel/<int:job_id>", methods=["POST"])
 @loginrequired

@@ 4,6 4,7 @@ from datetime import datetime
 from srht.config import cfg
 from srht.database import db
 from srht.email import send_email
+from srht.graphql import exec_gql
 from srht.oauth import UserType
 from srht.metrics import RedisQueueCollector
 from prometheus_client import Counter


@@ 22,20 23,15 @@ runner = Celery('builds', broker=builds_broker, config_source={
 builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue")
 builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted")
 
-def queue_build(job, manifest):
-    from buildsrht.types import JobStatus
-    job.status = JobStatus.queued
-    db.session.commit()
-    # crypto mining attempt
-    # pretend to accept it and let the admins know
-    sample = json.dumps(manifest.to_dict())
-    if "xuirig" in sample or "miner" in sample or "selci" in sample:
-        send_email(f"User {job.owner.canonical_name} attempted to submit cryptocurrency mining job #{job.id}",
-                cfg("sr.ht", "owner-email"),
-                "Cryptocurrency mining attempt on builds.sr.ht")
-    else:
-        builds_submitted.inc()
-        run_build.delay(job.id, manifest.to_dict())
+def submit_build(user, manifest, note=None, tags=[]):
+    resp = exec_gql("builds.sr.ht", """
+        mutation SubmitBuild($manifest: String!, $tags: [String!], $note: String) {
+            submit(manifest: $manifest, tags: $tags, note: $note) {
+                id
+            }
+        }
+    """, user=user, manifest=manifest, note=note, tags=tags)
+    return resp["submit"]["id"]
 
 def requires_payment(user):
     if allow_free:

@@ 1,6 1,6 @@
 #!/usr/bin/env python3
 from buildsrht.manifest import Manifest
-from buildsrht.runner import queue_build
+from buildsrht.runner import submit_build
 from buildsrht.types import Job, Task, User
 from getopt import getopt, GetoptError
 from srht.config import cfg, get_origin


@@ 43,18 43,10 @@ if cmd[0] == "submit":
         manifest = Manifest(yaml.safe_load(_manifest))
     except Exception as ex:
         fail(str(ex))
-    job = Job(user, _manifest)
-    job.image = manifest.image
-    job.note = " ".join([y for x, y in opts if x == "-n"]) or None
-    job.tags = "/".join([y for x, y in opts if x == "-t"]) or None
-    db.session.add(job)
-    db.session.flush()
-    for task in manifest.tasks:
-        t = Task(job, task.name)
-        db.session.add(t)
-        db.session.flush() # assigns IDs for ordering purposes
-    queue_build(job, manifest) # commits the session
-    url = f"{get_origin('builds.sr.ht', external=True)}/~{username}/job/{job.id}"
+    note = " ".join([y for x, y in opts if x == "-n"]) or None
+    tags = [y for x, y in opts if x == "-t"] or None
+    job_id = submit_build(user, _manifest, note=note, tags=tags)
+    url = f"{get_origin('builds.sr.ht', external=True)}/~{username}/job/{job_id}"
     print(url)
 else:
     fail(f"Unknown command {cmd[0]}")