Add flatpak nightly repository

.gitlab-ci.yml

@@ -88,7 +88,7 @@ build-flatpak-amd64:
   #image: 'registry.gitlab.gnome.org/gnome/gnome-runtime-images/gnome:master'
   tags: [docker]
   before_script:
-    - apt-get update && apt-get -y install flatpak-builder git python curl
+    - apt-get update && apt-get -y install flatpak-builder git python curl python3-aiohttp python3-tenacity gir1.2-ostree-1.0
     - flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
     - flatpak --noninteractive install --user flathub org.kde.Platform//5.15
     - flatpak --noninteractive install --user flathub org.kde.Sdk//5.15
@@ -99,6 +99,7 @@ build-flatpak-amd64:
     - flatpak-builder --user --disable-rofiles-fuse --ccache --repo=repo --default-branch=${CI_COMMIT_REF_NAME//\//_} --subject="Build of Nheko ${VERSION} `date`" app ../io.github.NhekoReborn.Nheko.json
     - flatpak build-bundle repo nheko-amd64.flatpak io.github.NhekoReborn.Nheko ${CI_COMMIT_REF_NAME//\//_}
   after_script:
+    - (cd ./scripts && ./upload-to-flatpak-repo.sh ../build-flatpak/repo) || true
     - bash ./.ci/upload-nightly-gitlab.sh build-flatpak/nheko-amd64.flatpak
   cache:
     key: "$CI_JOB_NAME"
@@ -115,7 +116,7 @@ build-flatpak-arm64:
   #image: 'registry.gitlab.gnome.org/gnome/gnome-runtime-images/gnome:master'
   tags: [docker-arm64]
   before_script:
-    - apt-get update && apt-get -y install flatpak-builder git python curl
+    - apt-get update && apt-get -y install flatpak-builder git python curl python3-aiohttp python3-tenacity gir1.2-ostree-1.0
     - flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
     - flatpak --noninteractive install --user flathub org.kde.Platform//5.15
     - flatpak --noninteractive install --user flathub org.kde.Sdk//5.15
@@ -126,6 +127,7 @@ build-flatpak-arm64:
     - flatpak-builder --user --disable-rofiles-fuse --ccache --repo=repo --default-branch=${CI_COMMIT_REF_NAME//\//_} --subject="Build of Nheko ${VERSION} `date` for arm64" app ../io.github.NhekoReborn.Nheko.json
     - flatpak build-bundle repo nheko-arm64.flatpak io.github.NhekoReborn.Nheko ${CI_COMMIT_REF_NAME//\//_}
   after_script:
+    - (cd ./scripts && ./upload-to-flatpak-repo.sh ../build-flatpak/repo) || true
     - bash ./.ci/upload-nightly-gitlab.sh build-flatpak/nheko-arm64.flatpak
   cache:
     key: "$CI_JOB_NAME"

README.md

@@ -4,6 +4,7 @@ nheko
 [![Build status](https://ci.appveyor.com/api/projects/status/07qrqbfylsg4hw2h/branch/master?svg=true)](https://ci.appveyor.com/project/redsky17/nheko/branch/master)
 [![Stable Version](https://img.shields.io/badge/download-stable-green.svg)](https://github.com/Nheko-Reborn/nheko/releases/v0.8.1)
 [![Nightly](https://img.shields.io/badge/download-nightly-green.svg)](https://matrix-static.neko.dev/room/!TshDrgpBNBDmfDeEGN:neko.dev/)
+[![Nightly Flatpak](https://img.shields.io/badge/download-nheko--nightly-green)](https://raw.githubusercontent.com/Nheko-Reborn/nheko/master/nheko-nightly.flatpakref)
 [![#nheko-reborn:matrix.org](https://img.shields.io/matrix/nheko-reborn:matrix.org.svg?label=%23nheko-reborn:matrix.org)](https://matrix.to/#/#nheko-reborn:matrix.org)
 [![AUR: nheko](https://img.shields.io/badge/AUR-nheko-blue.svg)](https://aur.archlinux.org/packages/nheko)
 <a href='https://flathub.org/apps/details/io.github.NhekoReborn.Nheko'><img width='240' alt='Download on Flathub' src='https://flathub.org/assets/badges/flathub-badge-en.png'/></a>

nheko-nightly.flatpakref

@@ -0,0 +1,10 @@
+[Flatpak Ref]
+Title=Nheko Nightly
+Name=io.github.NhekoReborn.Nheko
+Branch=master
+Url=https://flatpak.neko.dev/repo/nightly
+Homepage=https://nheko-reborn.github.io/
+Icon=https://nheko.im/nheko-reborn/nheko/-/raw/master/resources/nheko.svg
+RuntimeRepo=https://dl.flathub.org/repo/flathub.flatpakrepo
+IsRuntime=false
+GPGKey=mDMEXENMphYJKwYBBAHaRw8BAQdAqn+Eo42lPoGpJ5HaOf4nFGfxR0QtOggJTCfsdbOyL4e0Kk5pY29sYXMgV2VybmVyIDxuaWNvbGFzLndlcm5lckBob3RtYWlsLmRlPoiWBBMWCAA+FiEE1YtGJCWmo3Elxv7bkgauGyMeBbsFAlxDTVUCGwMFCQtJjooFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQkgauGyMeBbs2rQD/dAEoOGT21BL85A8LmPK743EboBAjoRbWcI1hHnvS28AA/3b3HYGwgvTC6hQLyz75zjpeO5ZaUtbezRyDUR4xabMAtCROaWNvbGFzIFdlcm5lciA8bmljb2xhc0BuZWtvZGV2Lm5ldD6IlgQTFggAPhYhBNWLRiQlpqNxJcb+25IGrhsjHgW7BQJcQ01GAhsDBQkLSY6KBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEJIGrhsjHgW7GxwBANT4gL03Uu9N5KmTBihC7B71+0r7M/azPbUh86NthCeIAQCF2JXa0axBKhgQF5fWC5ncL+m8ZpH7C5rzDqVgO82WALQnTmljb2xhcyBXZXJuZXIgPG5pY29sYXMud2VybmVyQGdteC5uZXQ+iJYEExYIAD4WIQTVi0YkJaajcSXG/tuSBq4bIx4FuwUCXENMpgIbAwUJC0mOigULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCSBq4bIx4FuxU5APoCRDYlJW0oTsJs3lcTTB5Nsqb3X4iCEDCjIgsA3wtsIwEAlGBzD8ElCYi2+8m8esSRNlmpRcGoqgXbceLxPUXFpQu4OARcQ0ymEgorBgEEAZdVAQUBAQdAD8dBmT3iqrqdlxSw90L0SIH11fVxiX9MdWfBkTi6PzUDAQgHiH4EGBYIACYWIQTVi0YkJaajcSXG/tuSBq4bIx4FuwUCXENMpgIbDAUJC0mOigAKCRCSBq4bIx4Fu/LNAQDhH64IBic6h7H3uvtSAFT4xNn7Epobt2baIaDp7uKsQQEAyI+oc5dLknABwIOMrQQuZCmGejx9e4/8HEqLCdszhgG4MwRgNICHFgkrBgEEAdpHDwEBB0DR9eFFzfR62FIi7g+txcQenLvKFzhlyTz0wo3icOy6RYj1BBgWCAAmFiEE1YtGJCWmo3Elxv7bkgauGyMeBbsFAmA0gIcCGwIFCQlmAYAAgQkQkgauGyMeBbt2IAQZFggAHRYhBGz14re9h4cNPaFEKMjXXmEHc/LZBQJgNICHAAoJEMjXXmEHc/LZhVMBAPdYRspdeFh6E9BDxGubT705e/pZFdCHjCToDyxgdW5KAP9sU0hFI5VDHD1h98RzxSt7hc3jxyPSzbG1MBUJ9gbfCVhcAPsFfeZc3v5UBgmn4uICFEGjlzAWCQ7WctE6QTSkY5aL/wD9ETJH5lB+i/8km/sOBKQozXR0yHHw46gB6ZWMeN1wfgq4MwRgNPutFgkrBgEEAdpHDwEBB0APwMn0FJmnAds8IO8iCl/RHr7fz8xnpGd7E4zVgCNZpIj1BBgWCAAmFiEE1YtGJCWmo3Elxv7bkgauGyMeBbsFAmA0+60CGwIFCQANLwAAgQkQkgauGyMeBbt2IAQZFggAHRYhBAH7QBkzNfVIZJM93RNnXzGtBKQcBQJgNPutAAoJEBNnXzGtBKQcHnUA/0E2H5sxmfZ+EWFTso3X4NWu3uN2xF+MdNaY8C72f9H6AP91XaNmlB9gV61rg6wcB5E/j0998yWS9gltY1XY1ImqDPvlAP4sHFs5zuDazgKYxZ/kFhENCgEStdpnvJjt/DxmQPVT3AD/QK5vGoMTIeYjihv0QCnnRDfboTTZHlaEqJW8i02PQww=

scripts/flat-manager-client

@@ -0,0 +1,766 @@
+#!/usr/bin/python3
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import asyncio
+import base64
+import binascii
+import errno
+import fnmatch
+import gzip
+import json
+import logging
+import os
+import sys
+import time
+import traceback
+from argparse import ArgumentParser
+from functools import reduce
+from urllib.parse import urljoin, urlparse, urlsplit, urlunparse, urlunsplit
+
+import aiohttp
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
+
+import gi
+gi.require_version('OSTree', '1.0')
+from gi.repository import Gio, GLib, OSTree
+
+UPLOAD_CHUNK_LIMIT = 4 * 1024 * 1024
+DEFAULT_LIMIT = 2 ** 16
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+class UsageException(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class ApiError(Exception):
+    def __init__(self, response, body):
+        self.url = response.url
+        self.status = response.status
+
+        try:
+            self.body = json.loads(response);
+        except:
+            self.body = {"status": self.status, "error-type": "no-error", "message": "No json error details from server"}
+
+    def repr(self):
+        return {
+            "type": "api",
+            "url": self.url,
+            "status_code": self.status,
+            "details": self.body
+        }
+
+    def __str__(self):
+        return "Api call to %s failed with status %d, details: %s" % (self.url, self.status, self.body)
+        return json.dumps(self.repr(), indent=4)
+
+# This is similar to the regular payload, but opens the file lazily
+class AsyncNamedFilePart(aiohttp.payload.Payload):
+    def __init__(self,
+                 value,
+                 disposition='attachment',
+                 *args,
+                 **kwargs):
+        self._file = None
+        if 'filename' not in kwargs:
+            kwargs['filename'] = os.path.basename(value)
+
+        super().__init__(value, *args, **kwargs)
+
+        if self._filename is not None and disposition is not None:
+            self.set_content_disposition(disposition, filename=self._filename, quote_fields=False)
+
+        self._size = os.stat(value).st_size
+
+    async def write(self, writer):
+        if self._file is None or self._file.closed:
+            self._file = open(self._value, 'rb')
+        try:
+            chunk = self._file.read(DEFAULT_LIMIT)
+            while chunk:
+                await writer.write(chunk)
+                chunk = self._file.read(DEFAULT_LIMIT)
+        finally:
+            self._file.close()
+
+    @property
+    def size(self):
+        return self._size
+
+def ostree_object_path(repo, obj):
+    repodir = repo.get_path().get_path()
+    return os.path.join(repodir, 'objects', obj[0:2], obj[2:])
+
+def ostree_get_dir_files(repo, objects, dirtree):
+    if dirtree.endswith(".dirtree"):
+        dirtree = dirtree[:-8]
+    dirtreev = repo.load_variant(OSTree.ObjectType.DIR_TREE, dirtree)[1]
+    iter = OSTree.RepoCommitTraverseIter()
+    iter.init_dirtree(repo, dirtreev, 0)
+    while True:
+        type = iter.next()
+        if type == OSTree.RepoCommitIterResult.END:
+            break
+        if type == OSTree.RepoCommitIterResult.ERROR:
+            break
+        if type == OSTree.RepoCommitIterResult.FILE:
+            d = iter.get_file()
+            objects.add(d.out_checksum + ".filez")
+        if type == OSTree.RepoCommitIterResult.DIR:
+            pass
+
+def local_needed_files(repo, metadata_objects):
+    objects = set()
+    for c in metadata_objects:
+        if c.endswith(".dirtree"):
+            ostree_get_dir_files(repo, objects, c)
+    return objects
+
+def local_needed_metadata_dirtree(repo, objects, dirtree_content, dirtree_meta):
+    objects.add(dirtree_meta + ".dirmeta")
+    dirtree_content_name = dirtree_content + ".dirtree"
+    if dirtree_content_name in objects:
+        return
+    objects.add(dirtree_content_name)
+
+    dirtreev = repo.load_variant(OSTree.ObjectType.DIR_TREE, dirtree_content)[1]
+    iter = OSTree.RepoCommitTraverseIter()
+    iter.init_dirtree(repo, dirtreev, 0)
+    while True:
+        type = iter.next()
+        if type == OSTree.RepoCommitIterResult.END:
+            break
+        if type == OSTree.RepoCommitIterResult.ERROR:
+            break
+        if type == OSTree.RepoCommitIterResult.FILE:
+            pass
+        if type == OSTree.RepoCommitIterResult.DIR:
+            d = iter.get_dir()
+            local_needed_metadata_dirtree(repo, objects, d.out_content_checksum, d.out_meta_checksum)
+
+def local_needed_metadata(repo, commits):
+    objects = set()
+    for rev in commits:
+        objects.add(rev + ".commit")
+        commitv = repo.load_variant(OSTree.ObjectType.COMMIT, rev)[1]
+        iter = OSTree.RepoCommitTraverseIter()
+        iter.init_commit(repo, commitv, 0)
+        while True:
+            type = iter.next()
+            if type == OSTree.RepoCommitIterResult.END:
+                break
+            if type == OSTree.RepoCommitIterResult.ERROR:
+                break
+            if type == OSTree.RepoCommitIterResult.FILE:
+                pass
+            if type == OSTree.RepoCommitIterResult.DIR:
+                d = iter.get_dir()
+                local_needed_metadata_dirtree(repo, objects, d.out_content_checksum, d.out_meta_checksum)
+    return objects
+
+
+def chunks(l, n):
+    """Yield successive n-sized chunks from l."""
+    for i in range(0, len(l), n):
+        yield l[i:i + n]
+
+async def missing_objects(session, build_url, token, wanted):
+    missing=[]
+    for chunk in chunks(wanted, 2000):
+        wanted_json=json.dumps({'wanted': chunk}).encode('utf-8')
+        data=gzip.compress(wanted_json)
+        headers = {
+            'Authorization': 'Bearer ' + token,
+            'Content-Encoding': 'gzip',
+            'Content-Type': 'application/json'
+        }
+        resp = await session.get(build_url + "/missing_objects", data=data, headers=headers)
+        async with resp:
+            if resp.status != 200:
+                raise ApiError(resp, await resp.text())
+            data = await resp.json()
+            missing.extend(data["missing"])
+    return missing
+
+@retry(
+    stop=stop_after_attempt(6),
+    wait=wait_fixed(20),
+    retry=retry_if_exception_type(ApiError),
+    reraise=True,
+)
+async def upload_files(session, build_url, token, files):
+    if len(files) == 0:
+        return
+    print("Uploading %d files (%d bytes)" % (len(files), reduce(lambda x, y: x + y, map(lambda f: f.size, files))))
+    with aiohttp.MultipartWriter() as writer:
+        for f in files:
+            writer.append(f)
+    writer.headers['Authorization'] = 'Bearer ' + token
+    resp = await session.request("post", build_url + '/upload', data=writer, headers=writer.headers)
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+async def upload_deltas(session, repo_path, build_url, token, deltas, refs, ignore_delta):
+    if not len(deltas):
+        return
+
+    req = []
+    for ref, commit in refs.items():
+        # Skip screenshots here
+        parts = ref.split("/")
+        if len(parts) == 4 and (parts[0] == "app" or parts[0] =="runtime") and not should_skip_delta(parts[1], ignore_delta):
+            for delta in deltas:
+                # Only upload from-scratch deltas, as these are the only reused ones
+                if delta == commit:
+                    print(" %s: %s" % (ref, delta))
+                    delta_name = delta_name_encode (delta)
+                    delta_dir = repo_path + "/deltas/" + delta_name[:2] + "/" + delta_name[2:]
+                    parts = os.listdir(delta_dir)
+                    for part in parts:
+                        req.append(AsyncNamedFilePart(delta_dir + "/" + part, filename = delta_name + "." + part + ".delta"))
+
+    if len(req):
+        await upload_files(session, build_url, token, req)
+
+
+async def upload_objects(session, repo_path, build_url, token, objects):
+    req = []
+    total_size = 0
+    for file_obj in objects:
+        named = get_object_multipart(repo_path, file_obj)
+        file_size = named.size
+        if total_size + file_size > UPLOAD_CHUNK_LIMIT: # The new object would bring us over the chunk limit
+            if len(req) > 0: # We already have some objects, upload those first
+                next_req = [named]
+                total_size = file_size
+            else:
+                next_req  =  []
+                req.append(named)
+                total_size = 0
+            await upload_files(session, build_url, token, req)
+            req = next_req
+        else:
+            total_size = total_size + file_size
+            req.append(named)
+
+    # Upload any remainder
+    await upload_files(session, build_url, token, req)
+
+async def create_ref(session, build_url, token, ref, commit):
+    print("Creating ref %s with commit %s" % (ref, commit))
+    resp = await session.post(build_url + "/build_ref", headers={'Authorization': 'Bearer ' + token}, json= { "ref": ref, "commit": commit} )
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        data = await resp.json()
+        return data
+
+async def add_extra_ids(session, build_url, token, extra_ids):
+    print("Adding extra ids %s" % (extra_ids))
+    resp = await session.post(build_url + "/add_extra_ids", headers={'Authorization': 'Bearer ' + token}, json= { "ids": extra_ids} )
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        data = await resp.json()
+        return data
+
+async def get_build(session, build_url, token):
+    resp = await session.get(build_url, headers={'Authorization': 'Bearer ' + token})
+    if resp.status != 200:
+        raise ApiError(resp, await resp.text())
+    data = await resp.json()
+    return data
+
+# For stupid reasons this is a string with json, lets expand it
+def reparse_job_results(job):
+    job["results"] = json.loads(job.get("results", "{}"))
+    return job
+
+async def get_job(session, job_url, token):
+    resp = await session.get(job_url, headers={'Authorization': 'Bearer ' + token}, json={})
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        data = await resp.json()
+        return data
+
+async def wait_for_job(session, job_url, token):
+    reported_delay = False
+    old_job_status  = 0
+    printed_len = 0
+    iterations_since_change=0
+    error_iterations = 0
+    while True:
+        try:
+            resp = await session.get(job_url, headers={'Authorization': 'Bearer ' + token}, json={'log-offset': printed_len})
+            async with resp:
+                if resp.status == 200:
+                    error_iterations = 0
+                    job = await resp.json()
+                    job_status = job['status']
+                    if job_status == 0 and not reported_delay:
+                        reported_delay = True
+                        start_after_struct = job.get("start_after", None)
+                        if start_after_struct:
+                            start_after = start_after_struct.get("secs_since_epoch", None)
+                            now = time.time()
+                            if start_after and start_after > now:
+                                print("Waiting %d seconds before starting job" % (int(start_after - now)))
+                    if job_status > 0 and old_job_status == 0:
+                        print("/ Job was started");
+                    old_job_status = job_status
+                    log = job['log']
+                    if len(log) > 0:
+                        iterations_since_change=0
+                        for line in log.splitlines(True):
+                            print("| %s" % line, end="")
+                        printed_len = printed_len + len(log)
+                    else:
+                        iterations_since_change=iterations_since_change+1
+                    if job_status > 1:
+                        if job_status == 2:
+                            print("\ Job completed successfully")
+                        else:
+                            print("\ Job failed")
+                        return job
+                else:
+                    iterations_since_change=4 # Start at 4 so we ramp up the delay faster
+                    error_iterations=error_iterations + 1
+                    if error_iterations <= 5:
+                        print("Unexpected response %s getting job log, ignoring" % resp.status)
+                    else:
+                        raise ApiError(resp, await resp.text())
+        except OSError as e:
+            if e.args[0] == errno.ECONNRESET:
+                # Client disconnected, retry
+                # Not sure exactly why, but i got a lot of ConnectionResetErrors here
+                # in tests. I guess the server stops reusing a http2 session after a bit
+                # Should be fine to retry with the backof
+                pass
+            else:
+                raise
+        # Some polling backoff to avoid loading the server
+        if iterations_since_change <= 1:
+            sleep_time=1
+        elif iterations_since_change < 5:
+            sleep_time=3
+        elif iterations_since_change < 15:
+            sleep_time=5
+        elif iterations_since_change < 30:
+            sleep_time=10
+        else:
+            sleep_time=60
+        time.sleep(sleep_time)
+
+async def commit_build(session, build_url, eol, eol_rebase, wait, token):
+    print("Committing build %s" % (build_url))
+    resp = await session.post(build_url + "/commit", headers={'Authorization': 'Bearer ' + token}, json= {
+        "endoflife": eol, "endoflife_rebase": eol_rebase
+    })
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        job = await resp.json()
+        job_url = resp.headers['location'];
+
+        if wait:
+            print("Waiting for commit job")
+            job = await wait_for_job(session, job_url, token);
+
+        reparse_job_results(job)
+        job["location"] = job_url
+        return job
+
+async def publish_build(session, build_url, wait, token):
+    print("Publishing build %s" % (build_url))
+    resp = await session.post(build_url + "/publish", headers={'Authorization': 'Bearer ' + token}, json= { } )
+    async with resp:
+        if resp.status == 400:
+            body = await resp.text()
+            try:
+                msg = json.loads(body)
+                if msg.get("current-state", "") == "published":
+                    print("the build has been already published")
+                    return {}
+            except:
+                pass
+
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        job = await resp.json()
+        job_url = resp.headers['location'];
+
+        if wait:
+            print("Waiting for publish job")
+            job = await wait_for_job(session, job_url, token);
+
+        reparse_job_results(job)
+        job["location"] = job_url
+        return job
+
+async def purge_build(session, build_url, token):
+    print("Purging build %s" % (build_url))
+    resp = await session.post(build_url + "/purge", headers={'Authorization': 'Bearer ' + token}, json= {} )
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        return await resp.json()
+
+async def create_token(session, manager_url, token, name, subject, scope, duration):
+    token_url = urljoin(manager_url, "/api/v1/token_subset")
+    resp = await session.post(token_url, headers={'Authorization': 'Bearer ' + token}, json = {
+        "name": name,
+        "sub": subject,
+        "scope": scope,
+        "duration": duration,
+    })
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        return await resp.json()
+
+def get_object_multipart(repo_path, object):
+    return AsyncNamedFilePart(repo_path + "/objects/" + object[:2] + "/" + object[2:], filename=object)
+
+async def create_command(session, args):
+    build_url = urljoin(args.manager_url, "/api/v1/build")
+    resp = await session.post(build_url, headers={'Authorization': 'Bearer ' + args.token}, json={
+        "repo": args.repo
+    })
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        data = await resp.json()
+        data["location"] = resp.headers['location']
+        if not args.print_output:
+            print(resp.headers['location'])
+        return data
+
+def delta_name_part_encode(commit):
+    return base64.b64encode(binascii.unhexlify(commit), b"+_")[:-1].decode("utf-8")
+
+def delta_name_encode (delta):
+    return "-".join(map(delta_name_part_encode, delta.split("-")))
+
+def should_skip_delta(id, globs):
+    if globs:
+        for glob in globs:
+            if fnmatch.fnmatch(id, glob):
+                return True
+    return False
+
+def build_url_to_api(build_url):
+    parts = urlparse(build_url)
+    path = os.path.dirname(os.path.dirname(parts.path))
+    return urlunparse((parts.scheme, parts.netloc, path, None, None, None))
+
+async def push_command(session, args):
+    local_repo = OSTree.Repo.new(Gio.File.new_for_path(args.repo_path))
+    try:
+        local_repo.open(None)
+    except GLib.Error as err:
+        raise UsageException("Can't open repo %s: %s" % (args.repo_path, err.message)) from err
+
+    refs = {}
+    if len(args.branches) == 0:
+        _, all_refs = local_repo.list_refs(None, None)
+        for ref in all_refs:
+            if ref.startswith("app/") or ref.startswith("runtime/") or ref.startswith("screenshots/"):
+                refs[ref] = all_refs[ref]
+    else:
+        for branch in args.branches:
+            _, rev = local_repo.resolve_rev(branch, False)
+            refs[branch] = rev
+
+    if (args.minimal_token):
+        id = os.path.basename(urlparse(args.build_url).path)
+        token = create_token(args.build_url, args.token, "minimal-upload", "build/%s" % (id), ["upload"], 60*60)["token"]
+    else:
+        token = args.token
+
+    print("Uploading refs to %s: %s"% (args.build_url, list(refs)))
+
+    metadata_objects = local_needed_metadata(local_repo, refs.values())
+
+    print("Refs contain %d metadata objects" % (len(metadata_objects)))
+
+    missing_metadata_objects = await missing_objects(session, args.build_url, token, list(metadata_objects))
+
+    print("Remote missing %d of those" % (len(missing_metadata_objects)))
+
+    file_objects = local_needed_files(local_repo, missing_metadata_objects)
+    print("Has %d file objects for those" % (len(file_objects)))
+
+    missing_file_objects = await missing_objects(session, args.build_url, token, list(file_objects))
+    print("Remote missing %d of those" % (len(missing_file_objects)))
+
+    # First upload all missing file objects
+    print("Uploading file objects")
+    await upload_objects(session, args.repo_path, args.build_url, token, missing_file_objects)
+
+    # Then all the metadata
+    print("Uploading metadata objects")
+    await upload_objects(session, args.repo_path, args.build_url, token, missing_metadata_objects)
+
+    _, deltas = local_repo.list_static_delta_names()
+    print("Uploading deltas")
+    await upload_deltas(session, args.repo_path, args.build_url, token, deltas, refs, args.ignore_delta)
+
+    # Then the refs
+    for ref, commit in refs.items():
+        await create_ref(session, args.build_url, token, ref, commit)
+
+    # Then any extra ids
+    if args.extra_id:
+        await add_extra_ids(session, args.build_url, token, args.extra_id)
+
+    commit_job = None
+    publish_job = None
+    update_job = None
+
+    # Note, this always uses the full token, as the minimal one only has upload permissions
+    if args.commit or args.publish:
+        commit_job = await commit_build(session, args.build_url, args.end_of_life, args.end_of_life_rebase, args.publish or args.wait, args.token)
+
+    if args.publish:
+        publish_job = await publish_build(session, args.build_url, args.wait or args.wait_update, args.token)
+        update_job_id = publish_job.get("results", {}).get("update-repo-job", None)
+        if update_job_id:
+            print("Queued repo update job %d" %(update_job_id))
+            update_job_url = build_url_to_api(args.build_url) + "/job/" + str(update_job_id)
+            if args.wait_update:
+                print("Waiting for repo update job")
+                update_job = await wait_for_job (session, update_job_url, token);
+            else:
+                update_job = await get_job(session, update_job_url, token)
+            reparse_job_results(update_job)
+            update_job["location"] = update_job_url
+
+    data = await get_build(session, args.build_url, args.token)
+    if commit_job:
+        data["commit_job"] = commit_job
+    if publish_job:
+        data["publish_job"] = publish_job
+    if update_job:
+        data["update_job"] = update_job
+    return data
+
+async def commit_command(session, args):
+    job = await commit_build(session, args.build_url, args.end_of_life, args.end_of_life_rebase, args.wait, args.token)
+    return job
+
+async def publish_command(session, args):
+    job = await publish_build(session, args.build_url, args.wait or args.wait_update, args.token)
+    update_job_id = job.get("results", {}).get("update-repo-job", None)
+    if update_job_id:
+        print("Queued repo update job %d" %(update_job_id))
+        update_job_url = build_url_to_api(args.build_url) + "/job/" + str(update_job_id)
+        if args.wait_update:
+            print("Waiting for repo update job")
+            update_job = await wait_for_job(session, update_job_url, args.token);
+        else:
+            update_job = await get_job(session, update_job_url, args.token)
+        reparse_job_results(update_job)
+        update_job["location"] = update_job_url
+    return job
+
+async def purge_command(session, args):
+    job = await purge_build(session, args.build_url, args.token)
+    return job
+
+async def create_token_command(session, args):
+    data = await create_token(session, args.manager_url, args.token, args.name, args.subject, args.scope, args.duration)
+    if not args.print_output:
+        print(data['token'])
+    return data
+
+async def follow_job_command(session, args):
+    job = await wait_for_job(session, args.job_url, args.token)
+    return job
+
+async def run_with_session(args):
+    timeout = aiohttp.ClientTimeout(total=90*60)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        result = await args.func(session, args)
+    return result
+
+if __name__ == '__main__':
+    progname = os.path.basename(sys.argv[0])
+
+    parser = ArgumentParser(prog=progname)
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='enable verbose output')
+    parser.add_argument('--debug', action='store_true',
+                        help='enable debugging output')
+    parser.add_argument('--output', help='Write output json to file')
+    parser.add_argument('--print-output', action='store_true', help='Print output json')
+    parser.add_argument('--token', help='use this token')
+    parser.add_argument('--token-file', help='use token from file')
+    subparsers = parser.add_subparsers(title='subcommands',
+                                       dest='subparser_name',
+                                       description='valid subcommands',
+                                       help='additional help')
+
+    create_parser = subparsers.add_parser('create', help='Create new build')
+    create_parser.add_argument('manager_url', help='remote repo manager url')
+    create_parser.add_argument('repo', help='repo name')
+    create_parser.set_defaults(func=create_command)
+
+    push_parser = subparsers.add_parser('push', help='Push to repo manager')
+    push_parser.add_argument('build_url', help='remote build url')
+    push_parser.add_argument('repo_path', help='local repository')
+    push_parser.add_argument('branches', nargs='*', help='branches to push')
+    push_parser.add_argument('--commit', action='store_true',
+                             help='commit build after pushing')
+    push_parser.add_argument('--publish', action='store_true',
+                             help='publish build after committing')
+    push_parser.add_argument('--extra-id', action='append', help='add extra collection-id')
+    push_parser.add_argument('--ignore-delta', action='append', help='don\'t upload deltas matching this glob')
+    push_parser.add_argument('--wait', action='store_true',
+                             help='wait for commit/publish to finish')
+    push_parser.add_argument('--wait-update', action='store_true',
+                             help='wait for update-repo to finish')
+    push_parser.add_argument('--minimal-token', action='store_true',
+                             help='Create minimal token for the upload')
+    push_parser.add_argument('--end-of-life', help='Set end of life')
+    push_parser.add_argument('--end-of-life-rebase', help='Set new ID which will supercede the current one')
+    push_parser.set_defaults(func=push_command)
+
+    commit_parser = subparsers.add_parser('commit', help='Commit build')
+    commit_parser.add_argument('--wait', action='store_true',
+                             help='wait for commit to finish')
+    commit_parser.add_argument('--end-of-life', help='Set end of life')
+    commit_parser.add_argument('--end-of-life-rebase', help='Set new ID which will supercede the current one')
+    commit_parser.add_argument('build_url', help='remote build url')
+    commit_parser.set_defaults(func=commit_command)
+
+    publish_parser = subparsers.add_parser('publish', help='Publish build')
+    publish_parser.add_argument('--wait', action='store_true',
+                             help='wait for publish to finish')
+    publish_parser.add_argument('--wait-update', action='store_true',
+                             help='wait for update-repo to finish')
+    publish_parser.add_argument('build_url', help='remote build url')
+    publish_parser.set_defaults(func=publish_command)
+
+    purge_parser = subparsers.add_parser('purge', help='Purge build')
+    purge_parser.add_argument('build_url', help='remote build url')
+    purge_parser.set_defaults(func=purge_command)
+
+    create_token_parser = subparsers.add_parser('create-token', help='Create subset token')
+    create_token_parser.add_argument('manager_url', help='remote repo manager url')
+    create_token_parser.add_argument('name', help='Name')
+    create_token_parser.add_argument('subject', help='Subject')
+    create_token_parser.add_argument('scope', nargs='*', help='Scope')
+    create_token_parser.add_argument('--duration', help='Duration until expires, in seconds',
+                                     default=60*60*24, # Default duration is one day
+                                     type=int)
+    create_token_parser.set_defaults(func=create_token_command)
+
+    follow_job_parser = subparsers.add_parser('follow-job', help='Follow existing job log')
+    follow_job_parser.add_argument('job_url', help='url of job')
+    follow_job_parser.set_defaults(func=follow_job_command)
+
+    args = parser.parse_args()
+
+    loglevel = logging.WARNING
+    if args.verbose:
+        loglevel = logging.INFO
+    if args.debug:
+        loglevel = logging.DEBUG
+
+    logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
+                        level=loglevel, stream=sys.stderr)
+
+    if not args.subparser_name:
+        print("No subcommand specified, see --help for usage")
+        exit(1)
+
+    if not args.token:
+        if args.token_file:
+            file = open(args.token_file, 'rb')
+            args.token = file.read().splitlines()[0].decode("utf-8").strip()
+        elif "REPO_TOKEN" in os.environ:
+            args.token = os.environ["REPO_TOKEN"]
+        else:
+            print("No token available, pass with --token, --token-file or $REPO_TOKEN")
+            exit(1)
+
+
+    res = 1
+    output = None
+    try:
+        loop = asyncio.get_event_loop()
+        result = loop.run_until_complete(run_with_session(args))
+
+        output = {
+            "command": args.subparser_name,
+            "result": result,
+        }
+        res = 0
+    except SystemExit:
+        # Something called sys.exit(), lets just exit
+        res = 1
+        raise # Pass on regular exit callse
+    except ApiError as e:
+        eprint(str(e))
+        output = {
+            "command": args.subparser_name,
+            "error": e.repr(),
+        }
+    except UsageException as e:
+        eprint(str(e))
+        output = {
+            "error": {
+                "type": "usage",
+                "details": {
+                    "message": str(e),
+                }
+            }
+        }
+    except:
+        ei = sys.exc_info()
+        eprint("Unexpected %s exception in %s: %s" % (ei[0].__name__, args.subparser_name, ei[1]))
+        eprint(traceback.format_exc())
+        output = {
+            "command": args.subparser_name,
+            "error": {
+                "type": "exception",
+                "details": {
+                    "error-type": ei[0].__name__,
+                    "message": str(ei[1]),
+                }
+            }
+        }
+        res = 1
+
+    if output:
+        if args.print_output:
+            print(json.dumps(output, indent=4))
+        if args.output:
+            f = open(args.output,"w+")
+            f.write(json.dumps(output, indent=4))
+            f.write("\n")
+            f.close()
+    exit(res)

scripts/upload-to-flatpak-repo.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+	echo "Missing repo to upload!"
+	exit 1
+fi
+
+if [ -n "${CI_COMMIT_TAG}" ]; then
+	BUILD_URL=$(./flat-manager-client create https://flatpak.neko.dev stable)
+elif [ "master" = "${CI_COMMIT_REF_NAME}" ]; then
+	BUILD_URL=$(./flat-manager-client create https://flatpak.neko.dev nightly)
+fi
+
+if [ -z "${BUILD_URL}" ]; then
+	echo "No upload to repo."
+	exit 0
+fi
+
+BUILD_URL=${BUILD_URL/http:/https:}
+
+./flat-manager-client push --commit $BUILD_URL $1
+./flat-manager-client publish $BUILD_URL
+