diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f03dcbbe..b44e740b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -88,7 +88,7 @@ build-flatpak-amd64: #image: 'registry.gitlab.gnome.org/gnome/gnome-runtime-images/gnome:master' tags: [docker] before_script: - - apt-get update && apt-get -y install flatpak-builder git python curl + - apt-get update && apt-get -y install flatpak-builder git python curl python3-aiohttp python3-tenacity gir1.2-ostree-1.0 - flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo - flatpak --noninteractive install --user flathub org.kde.Platform//5.15 - flatpak --noninteractive install --user flathub org.kde.Sdk//5.15 @@ -99,6 +99,7 @@ build-flatpak-amd64: - flatpak-builder --user --disable-rofiles-fuse --ccache --repo=repo --default-branch=${CI_COMMIT_REF_NAME//\//_} --subject="Build of Nheko ${VERSION} `date`" app ../io.github.NhekoReborn.Nheko.json - flatpak build-bundle repo nheko-amd64.flatpak io.github.NhekoReborn.Nheko ${CI_COMMIT_REF_NAME//\//_} after_script: + - (cd ./scripts && ./upload-to-flatpak-repo.sh ../build-flatpak/repo) || true - bash ./.ci/upload-nightly-gitlab.sh build-flatpak/nheko-amd64.flatpak cache: key: "$CI_JOB_NAME" @@ -115,7 +116,7 @@ build-flatpak-arm64: #image: 'registry.gitlab.gnome.org/gnome/gnome-runtime-images/gnome:master' tags: [docker-arm64] before_script: - - apt-get update && apt-get -y install flatpak-builder git python curl + - apt-get update && apt-get -y install flatpak-builder git python curl python3-aiohttp python3-tenacity gir1.2-ostree-1.0 - flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo - flatpak --noninteractive install --user flathub org.kde.Platform//5.15 - flatpak --noninteractive install --user flathub org.kde.Sdk//5.15 @@ -126,6 +127,7 @@ build-flatpak-arm64: - flatpak-builder --user --disable-rofiles-fuse --ccache --repo=repo --default-branch=${CI_COMMIT_REF_NAME//\//_} --subject="Build of Nheko ${VERSION} `date` for arm64" app ../io.github.NhekoReborn.Nheko.json - flatpak build-bundle repo nheko-arm64.flatpak io.github.NhekoReborn.Nheko ${CI_COMMIT_REF_NAME//\//_} after_script: + - (cd ./scripts && ./upload-to-flatpak-repo.sh ../build-flatpak/repo) || true - bash ./.ci/upload-nightly-gitlab.sh build-flatpak/nheko-arm64.flatpak cache: key: "$CI_JOB_NAME" diff --git a/README.md b/README.md index 15d4f416..4bcd7254 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ nheko [![Build status](https://ci.appveyor.com/api/projects/status/07qrqbfylsg4hw2h/branch/master?svg=true)](https://ci.appveyor.com/project/redsky17/nheko/branch/master) [![Stable Version](https://img.shields.io/badge/download-stable-green.svg)](https://github.com/Nheko-Reborn/nheko/releases/v0.8.1) [![Nightly](https://img.shields.io/badge/download-nightly-green.svg)](https://matrix-static.neko.dev/room/!TshDrgpBNBDmfDeEGN:neko.dev/) +[![Nightly Flatpak](https://img.shields.io/badge/download-nheko--nightly-green)](https://raw.githubusercontent.com/Nheko-Reborn/nheko/master/nheko-nightly.flatpakref) [![#nheko-reborn:matrix.org](https://img.shields.io/matrix/nheko-reborn:matrix.org.svg?label=%23nheko-reborn:matrix.org)](https://matrix.to/#/#nheko-reborn:matrix.org) [![AUR: nheko](https://img.shields.io/badge/AUR-nheko-blue.svg)](https://aur.archlinux.org/packages/nheko) Download on Flathub diff --git a/nheko-nightly.flatpakref b/nheko-nightly.flatpakref new file mode 100644 index 00000000..7d27bdfe --- /dev/null +++ b/nheko-nightly.flatpakref @@ -0,0 +1,10 @@ +[Flatpak Ref] +Title=Nheko Nightly +Name=io.github.NhekoReborn.Nheko +Branch=master +Url=https://flatpak.neko.dev/repo/nightly +Homepage=https://nheko-reborn.github.io/ +Icon=https://nheko.im/nheko-reborn/nheko/-/raw/master/resources/nheko.svg +RuntimeRepo=https://dl.flathub.org/repo/flathub.flatpakrepo +IsRuntime=false +GPGKey=mDMEXENMphYJKwYBBAHaRw8BAQdAqn+Eo42lPoGpJ5HaOf4nFGfxR0QtOggJTCfsdbOyL4e0Kk5pY29sYXMgV2VybmVyIDxuaWNvbGFzLndlcm5lckBob3RtYWlsLmRlPoiWBBMWCAA+FiEE1YtGJCWmo3Elxv7bkgauGyMeBbsFAlxDTVUCGwMFCQtJjooFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQkgauGyMeBbs2rQD/dAEoOGT21BL85A8LmPK743EboBAjoRbWcI1hHnvS28AA/3b3HYGwgvTC6hQLyz75zjpeO5ZaUtbezRyDUR4xabMAtCROaWNvbGFzIFdlcm5lciA8bmljb2xhc0BuZWtvZGV2Lm5ldD6IlgQTFggAPhYhBNWLRiQlpqNxJcb+25IGrhsjHgW7BQJcQ01GAhsDBQkLSY6KBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEJIGrhsjHgW7GxwBANT4gL03Uu9N5KmTBihC7B71+0r7M/azPbUh86NthCeIAQCF2JXa0axBKhgQF5fWC5ncL+m8ZpH7C5rzDqVgO82WALQnTmljb2xhcyBXZXJuZXIgPG5pY29sYXMud2VybmVyQGdteC5uZXQ+iJYEExYIAD4WIQTVi0YkJaajcSXG/tuSBq4bIx4FuwUCXENMpgIbAwUJC0mOigULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCSBq4bIx4FuxU5APoCRDYlJW0oTsJs3lcTTB5Nsqb3X4iCEDCjIgsA3wtsIwEAlGBzD8ElCYi2+8m8esSRNlmpRcGoqgXbceLxPUXFpQu4OARcQ0ymEgorBgEEAZdVAQUBAQdAD8dBmT3iqrqdlxSw90L0SIH11fVxiX9MdWfBkTi6PzUDAQgHiH4EGBYIACYWIQTVi0YkJaajcSXG/tuSBq4bIx4FuwUCXENMpgIbDAUJC0mOigAKCRCSBq4bIx4Fu/LNAQDhH64IBic6h7H3uvtSAFT4xNn7Epobt2baIaDp7uKsQQEAyI+oc5dLknABwIOMrQQuZCmGejx9e4/8HEqLCdszhgG4MwRgNICHFgkrBgEEAdpHDwEBB0DR9eFFzfR62FIi7g+txcQenLvKFzhlyTz0wo3icOy6RYj1BBgWCAAmFiEE1YtGJCWmo3Elxv7bkgauGyMeBbsFAmA0gIcCGwIFCQlmAYAAgQkQkgauGyMeBbt2IAQZFggAHRYhBGz14re9h4cNPaFEKMjXXmEHc/LZBQJgNICHAAoJEMjXXmEHc/LZhVMBAPdYRspdeFh6E9BDxGubT705e/pZFdCHjCToDyxgdW5KAP9sU0hFI5VDHD1h98RzxSt7hc3jxyPSzbG1MBUJ9gbfCVhcAPsFfeZc3v5UBgmn4uICFEGjlzAWCQ7WctE6QTSkY5aL/wD9ETJH5lB+i/8km/sOBKQozXR0yHHw46gB6ZWMeN1wfgq4MwRgNPutFgkrBgEEAdpHDwEBB0APwMn0FJmnAds8IO8iCl/RHr7fz8xnpGd7E4zVgCNZpIj1BBgWCAAmFiEE1YtGJCWmo3Elxv7bkgauGyMeBbsFAmA0+60CGwIFCQANLwAAgQkQkgauGyMeBbt2IAQZFggAHRYhBAH7QBkzNfVIZJM93RNnXzGtBKQcBQJgNPutAAoJEBNnXzGtBKQcHnUA/0E2H5sxmfZ+EWFTso3X4NWu3uN2xF+MdNaY8C72f9H6AP91XaNmlB9gV61rg6wcB5E/j0998yWS9gltY1XY1ImqDPvlAP4sHFs5zuDazgKYxZ/kFhENCgEStdpnvJjt/DxmQPVT3AD/QK5vGoMTIeYjihv0QCnnRDfboTTZHlaEqJW8i02PQww= diff --git a/scripts/flat-manager-client b/scripts/flat-manager-client new file mode 100755 index 00000000..183ebf81 --- /dev/null +++ b/scripts/flat-manager-client @@ -0,0 +1,766 @@ +#!/usr/bin/python3 + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import asyncio +import base64 +import binascii +import errno +import fnmatch +import gzip +import json +import logging +import os +import sys +import time +import traceback +from argparse import ArgumentParser +from functools import reduce +from urllib.parse import urljoin, urlparse, urlsplit, urlunparse, urlunsplit + +import aiohttp +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed + +import gi +gi.require_version('OSTree', '1.0') +from gi.repository import Gio, GLib, OSTree + +UPLOAD_CHUNK_LIMIT = 4 * 1024 * 1024 +DEFAULT_LIMIT = 2 ** 16 + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +class UsageException(Exception): + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class ApiError(Exception): + def __init__(self, response, body): + self.url = response.url + self.status = response.status + + try: + self.body = json.loads(response); + except: + self.body = {"status": self.status, "error-type": "no-error", "message": "No json error details from server"} + + def repr(self): + return { + "type": "api", + "url": self.url, + "status_code": self.status, + "details": self.body + } + + def __str__(self): + return "Api call to %s failed with status %d, details: %s" % (self.url, self.status, self.body) + return json.dumps(self.repr(), indent=4) + +# This is similar to the regular payload, but opens the file lazily +class AsyncNamedFilePart(aiohttp.payload.Payload): + def __init__(self, + value, + disposition='attachment', + *args, + **kwargs): + self._file = None + if 'filename' not in kwargs: + kwargs['filename'] = os.path.basename(value) + + super().__init__(value, *args, **kwargs) + + if self._filename is not None and disposition is not None: + self.set_content_disposition(disposition, filename=self._filename, quote_fields=False) + + self._size = os.stat(value).st_size + + async def write(self, writer): + if self._file is None or self._file.closed: + self._file = open(self._value, 'rb') + try: + chunk = self._file.read(DEFAULT_LIMIT) + while chunk: + await writer.write(chunk) + chunk = self._file.read(DEFAULT_LIMIT) + finally: + self._file.close() + + @property + def size(self): + return self._size + +def ostree_object_path(repo, obj): + repodir = repo.get_path().get_path() + return os.path.join(repodir, 'objects', obj[0:2], obj[2:]) + +def ostree_get_dir_files(repo, objects, dirtree): + if dirtree.endswith(".dirtree"): + dirtree = dirtree[:-8] + dirtreev = repo.load_variant(OSTree.ObjectType.DIR_TREE, dirtree)[1] + iter = OSTree.RepoCommitTraverseIter() + iter.init_dirtree(repo, dirtreev, 0) + while True: + type = iter.next() + if type == OSTree.RepoCommitIterResult.END: + break + if type == OSTree.RepoCommitIterResult.ERROR: + break + if type == OSTree.RepoCommitIterResult.FILE: + d = iter.get_file() + objects.add(d.out_checksum + ".filez") + if type == OSTree.RepoCommitIterResult.DIR: + pass + +def local_needed_files(repo, metadata_objects): + objects = set() + for c in metadata_objects: + if c.endswith(".dirtree"): + ostree_get_dir_files(repo, objects, c) + return objects + +def local_needed_metadata_dirtree(repo, objects, dirtree_content, dirtree_meta): + objects.add(dirtree_meta + ".dirmeta") + dirtree_content_name = dirtree_content + ".dirtree" + if dirtree_content_name in objects: + return + objects.add(dirtree_content_name) + + dirtreev = repo.load_variant(OSTree.ObjectType.DIR_TREE, dirtree_content)[1] + iter = OSTree.RepoCommitTraverseIter() + iter.init_dirtree(repo, dirtreev, 0) + while True: + type = iter.next() + if type == OSTree.RepoCommitIterResult.END: + break + if type == OSTree.RepoCommitIterResult.ERROR: + break + if type == OSTree.RepoCommitIterResult.FILE: + pass + if type == OSTree.RepoCommitIterResult.DIR: + d = iter.get_dir() + local_needed_metadata_dirtree(repo, objects, d.out_content_checksum, d.out_meta_checksum) + +def local_needed_metadata(repo, commits): + objects = set() + for rev in commits: + objects.add(rev + ".commit") + commitv = repo.load_variant(OSTree.ObjectType.COMMIT, rev)[1] + iter = OSTree.RepoCommitTraverseIter() + iter.init_commit(repo, commitv, 0) + while True: + type = iter.next() + if type == OSTree.RepoCommitIterResult.END: + break + if type == OSTree.RepoCommitIterResult.ERROR: + break + if type == OSTree.RepoCommitIterResult.FILE: + pass + if type == OSTree.RepoCommitIterResult.DIR: + d = iter.get_dir() + local_needed_metadata_dirtree(repo, objects, d.out_content_checksum, d.out_meta_checksum) + return objects + + +def chunks(l, n): + """Yield successive n-sized chunks from l.""" + for i in range(0, len(l), n): + yield l[i:i + n] + +async def missing_objects(session, build_url, token, wanted): + missing=[] + for chunk in chunks(wanted, 2000): + wanted_json=json.dumps({'wanted': chunk}).encode('utf-8') + data=gzip.compress(wanted_json) + headers = { + 'Authorization': 'Bearer ' + token, + 'Content-Encoding': 'gzip', + 'Content-Type': 'application/json' + } + resp = await session.get(build_url + "/missing_objects", data=data, headers=headers) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + data = await resp.json() + missing.extend(data["missing"]) + return missing + +@retry( + stop=stop_after_attempt(6), + wait=wait_fixed(20), + retry=retry_if_exception_type(ApiError), + reraise=True, +) +async def upload_files(session, build_url, token, files): + if len(files) == 0: + return + print("Uploading %d files (%d bytes)" % (len(files), reduce(lambda x, y: x + y, map(lambda f: f.size, files)))) + with aiohttp.MultipartWriter() as writer: + for f in files: + writer.append(f) + writer.headers['Authorization'] = 'Bearer ' + token + resp = await session.request("post", build_url + '/upload', data=writer, headers=writer.headers) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + +async def upload_deltas(session, repo_path, build_url, token, deltas, refs, ignore_delta): + if not len(deltas): + return + + req = [] + for ref, commit in refs.items(): + # Skip screenshots here + parts = ref.split("/") + if len(parts) == 4 and (parts[0] == "app" or parts[0] =="runtime") and not should_skip_delta(parts[1], ignore_delta): + for delta in deltas: + # Only upload from-scratch deltas, as these are the only reused ones + if delta == commit: + print(" %s: %s" % (ref, delta)) + delta_name = delta_name_encode (delta) + delta_dir = repo_path + "/deltas/" + delta_name[:2] + "/" + delta_name[2:] + parts = os.listdir(delta_dir) + for part in parts: + req.append(AsyncNamedFilePart(delta_dir + "/" + part, filename = delta_name + "." + part + ".delta")) + + if len(req): + await upload_files(session, build_url, token, req) + + +async def upload_objects(session, repo_path, build_url, token, objects): + req = [] + total_size = 0 + for file_obj in objects: + named = get_object_multipart(repo_path, file_obj) + file_size = named.size + if total_size + file_size > UPLOAD_CHUNK_LIMIT: # The new object would bring us over the chunk limit + if len(req) > 0: # We already have some objects, upload those first + next_req = [named] + total_size = file_size + else: + next_req = [] + req.append(named) + total_size = 0 + await upload_files(session, build_url, token, req) + req = next_req + else: + total_size = total_size + file_size + req.append(named) + + # Upload any remainder + await upload_files(session, build_url, token, req) + +async def create_ref(session, build_url, token, ref, commit): + print("Creating ref %s with commit %s" % (ref, commit)) + resp = await session.post(build_url + "/build_ref", headers={'Authorization': 'Bearer ' + token}, json= { "ref": ref, "commit": commit} ) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + + data = await resp.json() + return data + +async def add_extra_ids(session, build_url, token, extra_ids): + print("Adding extra ids %s" % (extra_ids)) + resp = await session.post(build_url + "/add_extra_ids", headers={'Authorization': 'Bearer ' + token}, json= { "ids": extra_ids} ) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + + data = await resp.json() + return data + +async def get_build(session, build_url, token): + resp = await session.get(build_url, headers={'Authorization': 'Bearer ' + token}) + if resp.status != 200: + raise ApiError(resp, await resp.text()) + data = await resp.json() + return data + +# For stupid reasons this is a string with json, lets expand it +def reparse_job_results(job): + job["results"] = json.loads(job.get("results", "{}")) + return job + +async def get_job(session, job_url, token): + resp = await session.get(job_url, headers={'Authorization': 'Bearer ' + token}, json={}) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + data = await resp.json() + return data + +async def wait_for_job(session, job_url, token): + reported_delay = False + old_job_status = 0 + printed_len = 0 + iterations_since_change=0 + error_iterations = 0 + while True: + try: + resp = await session.get(job_url, headers={'Authorization': 'Bearer ' + token}, json={'log-offset': printed_len}) + async with resp: + if resp.status == 200: + error_iterations = 0 + job = await resp.json() + job_status = job['status'] + if job_status == 0 and not reported_delay: + reported_delay = True + start_after_struct = job.get("start_after", None) + if start_after_struct: + start_after = start_after_struct.get("secs_since_epoch", None) + now = time.time() + if start_after and start_after > now: + print("Waiting %d seconds before starting job" % (int(start_after - now))) + if job_status > 0 and old_job_status == 0: + print("/ Job was started"); + old_job_status = job_status + log = job['log'] + if len(log) > 0: + iterations_since_change=0 + for line in log.splitlines(True): + print("| %s" % line, end="") + printed_len = printed_len + len(log) + else: + iterations_since_change=iterations_since_change+1 + if job_status > 1: + if job_status == 2: + print("\ Job completed successfully") + else: + print("\ Job failed") + return job + else: + iterations_since_change=4 # Start at 4 so we ramp up the delay faster + error_iterations=error_iterations + 1 + if error_iterations <= 5: + print("Unexpected response %s getting job log, ignoring" % resp.status) + else: + raise ApiError(resp, await resp.text()) + except OSError as e: + if e.args[0] == errno.ECONNRESET: + # Client disconnected, retry + # Not sure exactly why, but i got a lot of ConnectionResetErrors here + # in tests. I guess the server stops reusing a http2 session after a bit + # Should be fine to retry with the backof + pass + else: + raise + # Some polling backoff to avoid loading the server + if iterations_since_change <= 1: + sleep_time=1 + elif iterations_since_change < 5: + sleep_time=3 + elif iterations_since_change < 15: + sleep_time=5 + elif iterations_since_change < 30: + sleep_time=10 + else: + sleep_time=60 + time.sleep(sleep_time) + +async def commit_build(session, build_url, eol, eol_rebase, wait, token): + print("Committing build %s" % (build_url)) + resp = await session.post(build_url + "/commit", headers={'Authorization': 'Bearer ' + token}, json= { + "endoflife": eol, "endoflife_rebase": eol_rebase + }) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + + job = await resp.json() + job_url = resp.headers['location']; + + if wait: + print("Waiting for commit job") + job = await wait_for_job(session, job_url, token); + + reparse_job_results(job) + job["location"] = job_url + return job + +async def publish_build(session, build_url, wait, token): + print("Publishing build %s" % (build_url)) + resp = await session.post(build_url + "/publish", headers={'Authorization': 'Bearer ' + token}, json= { } ) + async with resp: + if resp.status == 400: + body = await resp.text() + try: + msg = json.loads(body) + if msg.get("current-state", "") == "published": + print("the build has been already published") + return {} + except: + pass + + if resp.status != 200: + raise ApiError(resp, await resp.text()) + + job = await resp.json() + job_url = resp.headers['location']; + + if wait: + print("Waiting for publish job") + job = await wait_for_job(session, job_url, token); + + reparse_job_results(job) + job["location"] = job_url + return job + +async def purge_build(session, build_url, token): + print("Purging build %s" % (build_url)) + resp = await session.post(build_url + "/purge", headers={'Authorization': 'Bearer ' + token}, json= {} ) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + return await resp.json() + +async def create_token(session, manager_url, token, name, subject, scope, duration): + token_url = urljoin(manager_url, "/api/v1/token_subset") + resp = await session.post(token_url, headers={'Authorization': 'Bearer ' + token}, json = { + "name": name, + "sub": subject, + "scope": scope, + "duration": duration, + }) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + return await resp.json() + +def get_object_multipart(repo_path, object): + return AsyncNamedFilePart(repo_path + "/objects/" + object[:2] + "/" + object[2:], filename=object) + +async def create_command(session, args): + build_url = urljoin(args.manager_url, "/api/v1/build") + resp = await session.post(build_url, headers={'Authorization': 'Bearer ' + args.token}, json={ + "repo": args.repo + }) + async with resp: + if resp.status != 200: + raise ApiError(resp, await resp.text()) + data = await resp.json() + data["location"] = resp.headers['location'] + if not args.print_output: + print(resp.headers['location']) + return data + +def delta_name_part_encode(commit): + return base64.b64encode(binascii.unhexlify(commit), b"+_")[:-1].decode("utf-8") + +def delta_name_encode (delta): + return "-".join(map(delta_name_part_encode, delta.split("-"))) + +def should_skip_delta(id, globs): + if globs: + for glob in globs: + if fnmatch.fnmatch(id, glob): + return True + return False + +def build_url_to_api(build_url): + parts = urlparse(build_url) + path = os.path.dirname(os.path.dirname(parts.path)) + return urlunparse((parts.scheme, parts.netloc, path, None, None, None)) + +async def push_command(session, args): + local_repo = OSTree.Repo.new(Gio.File.new_for_path(args.repo_path)) + try: + local_repo.open(None) + except GLib.Error as err: + raise UsageException("Can't open repo %s: %s" % (args.repo_path, err.message)) from err + + refs = {} + if len(args.branches) == 0: + _, all_refs = local_repo.list_refs(None, None) + for ref in all_refs: + if ref.startswith("app/") or ref.startswith("runtime/") or ref.startswith("screenshots/"): + refs[ref] = all_refs[ref] + else: + for branch in args.branches: + _, rev = local_repo.resolve_rev(branch, False) + refs[branch] = rev + + if (args.minimal_token): + id = os.path.basename(urlparse(args.build_url).path) + token = create_token(args.build_url, args.token, "minimal-upload", "build/%s" % (id), ["upload"], 60*60)["token"] + else: + token = args.token + + print("Uploading refs to %s: %s"% (args.build_url, list(refs))) + + metadata_objects = local_needed_metadata(local_repo, refs.values()) + + print("Refs contain %d metadata objects" % (len(metadata_objects))) + + missing_metadata_objects = await missing_objects(session, args.build_url, token, list(metadata_objects)) + + print("Remote missing %d of those" % (len(missing_metadata_objects))) + + file_objects = local_needed_files(local_repo, missing_metadata_objects) + print("Has %d file objects for those" % (len(file_objects))) + + missing_file_objects = await missing_objects(session, args.build_url, token, list(file_objects)) + print("Remote missing %d of those" % (len(missing_file_objects))) + + # First upload all missing file objects + print("Uploading file objects") + await upload_objects(session, args.repo_path, args.build_url, token, missing_file_objects) + + # Then all the metadata + print("Uploading metadata objects") + await upload_objects(session, args.repo_path, args.build_url, token, missing_metadata_objects) + + _, deltas = local_repo.list_static_delta_names() + print("Uploading deltas") + await upload_deltas(session, args.repo_path, args.build_url, token, deltas, refs, args.ignore_delta) + + # Then the refs + for ref, commit in refs.items(): + await create_ref(session, args.build_url, token, ref, commit) + + # Then any extra ids + if args.extra_id: + await add_extra_ids(session, args.build_url, token, args.extra_id) + + commit_job = None + publish_job = None + update_job = None + + # Note, this always uses the full token, as the minimal one only has upload permissions + if args.commit or args.publish: + commit_job = await commit_build(session, args.build_url, args.end_of_life, args.end_of_life_rebase, args.publish or args.wait, args.token) + + if args.publish: + publish_job = await publish_build(session, args.build_url, args.wait or args.wait_update, args.token) + update_job_id = publish_job.get("results", {}).get("update-repo-job", None) + if update_job_id: + print("Queued repo update job %d" %(update_job_id)) + update_job_url = build_url_to_api(args.build_url) + "/job/" + str(update_job_id) + if args.wait_update: + print("Waiting for repo update job") + update_job = await wait_for_job (session, update_job_url, token); + else: + update_job = await get_job(session, update_job_url, token) + reparse_job_results(update_job) + update_job["location"] = update_job_url + + data = await get_build(session, args.build_url, args.token) + if commit_job: + data["commit_job"] = commit_job + if publish_job: + data["publish_job"] = publish_job + if update_job: + data["update_job"] = update_job + return data + +async def commit_command(session, args): + job = await commit_build(session, args.build_url, args.end_of_life, args.end_of_life_rebase, args.wait, args.token) + return job + +async def publish_command(session, args): + job = await publish_build(session, args.build_url, args.wait or args.wait_update, args.token) + update_job_id = job.get("results", {}).get("update-repo-job", None) + if update_job_id: + print("Queued repo update job %d" %(update_job_id)) + update_job_url = build_url_to_api(args.build_url) + "/job/" + str(update_job_id) + if args.wait_update: + print("Waiting for repo update job") + update_job = await wait_for_job(session, update_job_url, args.token); + else: + update_job = await get_job(session, update_job_url, args.token) + reparse_job_results(update_job) + update_job["location"] = update_job_url + return job + +async def purge_command(session, args): + job = await purge_build(session, args.build_url, args.token) + return job + +async def create_token_command(session, args): + data = await create_token(session, args.manager_url, args.token, args.name, args.subject, args.scope, args.duration) + if not args.print_output: + print(data['token']) + return data + +async def follow_job_command(session, args): + job = await wait_for_job(session, args.job_url, args.token) + return job + +async def run_with_session(args): + timeout = aiohttp.ClientTimeout(total=90*60) + async with aiohttp.ClientSession(timeout=timeout) as session: + result = await args.func(session, args) + return result + +if __name__ == '__main__': + progname = os.path.basename(sys.argv[0]) + + parser = ArgumentParser(prog=progname) + parser.add_argument('-v', '--verbose', action='store_true', + help='enable verbose output') + parser.add_argument('--debug', action='store_true', + help='enable debugging output') + parser.add_argument('--output', help='Write output json to file') + parser.add_argument('--print-output', action='store_true', help='Print output json') + parser.add_argument('--token', help='use this token') + parser.add_argument('--token-file', help='use token from file') + subparsers = parser.add_subparsers(title='subcommands', + dest='subparser_name', + description='valid subcommands', + help='additional help') + + create_parser = subparsers.add_parser('create', help='Create new build') + create_parser.add_argument('manager_url', help='remote repo manager url') + create_parser.add_argument('repo', help='repo name') + create_parser.set_defaults(func=create_command) + + push_parser = subparsers.add_parser('push', help='Push to repo manager') + push_parser.add_argument('build_url', help='remote build url') + push_parser.add_argument('repo_path', help='local repository') + push_parser.add_argument('branches', nargs='*', help='branches to push') + push_parser.add_argument('--commit', action='store_true', + help='commit build after pushing') + push_parser.add_argument('--publish', action='store_true', + help='publish build after committing') + push_parser.add_argument('--extra-id', action='append', help='add extra collection-id') + push_parser.add_argument('--ignore-delta', action='append', help='don\'t upload deltas matching this glob') + push_parser.add_argument('--wait', action='store_true', + help='wait for commit/publish to finish') + push_parser.add_argument('--wait-update', action='store_true', + help='wait for update-repo to finish') + push_parser.add_argument('--minimal-token', action='store_true', + help='Create minimal token for the upload') + push_parser.add_argument('--end-of-life', help='Set end of life') + push_parser.add_argument('--end-of-life-rebase', help='Set new ID which will supercede the current one') + push_parser.set_defaults(func=push_command) + + commit_parser = subparsers.add_parser('commit', help='Commit build') + commit_parser.add_argument('--wait', action='store_true', + help='wait for commit to finish') + commit_parser.add_argument('--end-of-life', help='Set end of life') + commit_parser.add_argument('--end-of-life-rebase', help='Set new ID which will supercede the current one') + commit_parser.add_argument('build_url', help='remote build url') + commit_parser.set_defaults(func=commit_command) + + publish_parser = subparsers.add_parser('publish', help='Publish build') + publish_parser.add_argument('--wait', action='store_true', + help='wait for publish to finish') + publish_parser.add_argument('--wait-update', action='store_true', + help='wait for update-repo to finish') + publish_parser.add_argument('build_url', help='remote build url') + publish_parser.set_defaults(func=publish_command) + + purge_parser = subparsers.add_parser('purge', help='Purge build') + purge_parser.add_argument('build_url', help='remote build url') + purge_parser.set_defaults(func=purge_command) + + create_token_parser = subparsers.add_parser('create-token', help='Create subset token') + create_token_parser.add_argument('manager_url', help='remote repo manager url') + create_token_parser.add_argument('name', help='Name') + create_token_parser.add_argument('subject', help='Subject') + create_token_parser.add_argument('scope', nargs='*', help='Scope') + create_token_parser.add_argument('--duration', help='Duration until expires, in seconds', + default=60*60*24, # Default duration is one day + type=int) + create_token_parser.set_defaults(func=create_token_command) + + follow_job_parser = subparsers.add_parser('follow-job', help='Follow existing job log') + follow_job_parser.add_argument('job_url', help='url of job') + follow_job_parser.set_defaults(func=follow_job_command) + + args = parser.parse_args() + + loglevel = logging.WARNING + if args.verbose: + loglevel = logging.INFO + if args.debug: + loglevel = logging.DEBUG + + logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s', + level=loglevel, stream=sys.stderr) + + if not args.subparser_name: + print("No subcommand specified, see --help for usage") + exit(1) + + if not args.token: + if args.token_file: + file = open(args.token_file, 'rb') + args.token = file.read().splitlines()[0].decode("utf-8").strip() + elif "REPO_TOKEN" in os.environ: + args.token = os.environ["REPO_TOKEN"] + else: + print("No token available, pass with --token, --token-file or $REPO_TOKEN") + exit(1) + + + res = 1 + output = None + try: + loop = asyncio.get_event_loop() + result = loop.run_until_complete(run_with_session(args)) + + output = { + "command": args.subparser_name, + "result": result, + } + res = 0 + except SystemExit: + # Something called sys.exit(), lets just exit + res = 1 + raise # Pass on regular exit callse + except ApiError as e: + eprint(str(e)) + output = { + "command": args.subparser_name, + "error": e.repr(), + } + except UsageException as e: + eprint(str(e)) + output = { + "error": { + "type": "usage", + "details": { + "message": str(e), + } + } + } + except: + ei = sys.exc_info() + eprint("Unexpected %s exception in %s: %s" % (ei[0].__name__, args.subparser_name, ei[1])) + eprint(traceback.format_exc()) + output = { + "command": args.subparser_name, + "error": { + "type": "exception", + "details": { + "error-type": ei[0].__name__, + "message": str(ei[1]), + } + } + } + res = 1 + + if output: + if args.print_output: + print(json.dumps(output, indent=4)) + if args.output: + f = open(args.output,"w+") + f.write(json.dumps(output, indent=4)) + f.write("\n") + f.close() + exit(res) diff --git a/scripts/upload-to-flatpak-repo.sh b/scripts/upload-to-flatpak-repo.sh new file mode 100755 index 00000000..262a1635 --- /dev/null +++ b/scripts/upload-to-flatpak-repo.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "Missing repo to upload!" + exit 1 +fi + +if [ -n "${CI_COMMIT_TAG}" ]; then + BUILD_URL=$(./flat-manager-client create https://flatpak.neko.dev stable) +elif [ "master" = "${CI_COMMIT_REF_NAME}" ]; then + BUILD_URL=$(./flat-manager-client create https://flatpak.neko.dev nightly) +fi + +if [ -z "${BUILD_URL}" ]; then + echo "No upload to repo." + exit 0 +fi + +BUILD_URL=${BUILD_URL/http:/https:} + +./flat-manager-client push --commit $BUILD_URL $1 +./flat-manager-client publish $BUILD_URL +