Update

ci/gitlab/build_manifest.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+import subprocess
+import urllib.error
+import urllib.parse
+import urllib.request
+import json
+
+from typing import Dict, Tuple, List
+# from pprint import pprint
+
+if sys.version_info < (3, 6):
+    raise SystemExit('Need Python 3.6 or newer')
+
+GSTREAMER_MODULES: List[str] = [
+    'orc',
+    'cerbero',
+    'gst-build',
+    'gstreamer',
+    'gst-plugins-base',
+    'gst-plugins-good',
+    'gst-plugins-bad',
+    'gst-plugins-ugly',
+    'gst-libav',
+    'gst-devtools',
+    'gst-docs',
+    'gst-editing-services',
+    'gst-omx',
+    'gst-python',
+    'gst-rtsp-server',
+    'gstreamer-sharp',
+    'gstreamer-vaapi',
+    'gst-integration-testsuites',
+    'gst-examples',
+]
+
+MANIFEST_TEMPLATE: str = """<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote fetch="{}" name="user"/>
+  <remote fetch="https://gitlab.freedesktop.org/gstreamer/" name="origin"/>
+{}
+</manifest>"""
+
+
+CERBERO_DEPS_LOGS_TARGETS = (
+    ('cross-ios', 'universal'),
+    ('cross-windows-mingw', 'x86'),
+    ('cross-windows-mingw', 'x86_64'),
+    ('cross-android', 'universal'),
+    ('fedora', 'x86_64'),
+    ('macos', 'x86_64'),
+    ('windows-msvc', 'x86_64'),
+)
+
+# Disallow git prompting for a username/password
+os.environ['GIT_TERMINAL_PROMPT'] = '0'
+def git(*args, repository_path='.'):
+    return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
+
+def get_cerbero_last_build_info (branch : str):
+    # Fetch the deps log for all (distro, arch) targets
+    all_commits = {}
+    for distro, arch in CERBERO_DEPS_LOGS_TARGETS:
+        url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log'
+        print(f'Fetching {url}')
+        try:
+            req = urllib.request.Request(url)
+            resp = urllib.request.urlopen(req);
+            deps = json.loads(resp.read())
+        except urllib.error.URLError as e:
+            print(f'WARNING: Failed to GET {url}: {e!s}')
+            continue
+
+        for dep in deps:
+            commit = dep['commit']
+            if commit not in all_commits:
+                all_commits[commit] = []
+            all_commits[commit].append((distro, arch))
+
+    # Fetch the cerbero commit that has the most number of caches
+    best_commit = None
+    newest_commit = None
+    max_caches = 0
+    total_caches = len(CERBERO_DEPS_LOGS_TARGETS)
+    for commit, targets in all_commits.items():
+        if newest_commit is None:
+            newest_commit = commit
+        have_caches = len(targets)
+        # If this commit has caches for all targets, just use it
+        if have_caches == total_caches:
+            best_commit = commit
+            break
+        # Else, try to find the commit with the most caches
+        if have_caches > max_caches:
+            max_caches = have_caches
+            best_commit = commit
+    if newest_commit is None:
+        print('WARNING: No deps logs were found, will build from scratch')
+    if best_commit != newest_commit:
+        print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead')
+    return best_commit
+
+
+def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
+    try:
+        res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
+    except subprocess.CalledProcessError:
+        return None, None
+
+    if not res:
+        return None, None
+
+    # Special case cerbero to avoid cache misses
+    if module == 'cerbero':
+        sha = get_cerbero_last_build_info(branch)
+        if sha is not None:
+            return sha, sha
+
+    lines = res.split('\n')
+    for line in lines:
+        if line.endswith('/' + branch):
+            try:
+                sha, refname = line.split('\t')
+            except ValueError:
+                continue
+            return refname.strip(), sha
+
+    return None, None
+
+
+def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
+    namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
+    ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
+
+    if module == "orc":
+        ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
+
+    if module == os.environ['CI_PROJECT_NAME']:
+        return 'user', branchname, os.environ['CI_COMMIT_SHA']
+
+    if branchname != ups_branch:
+        remote_refname, sha = get_branch_info(module, namespace, branchname)
+        if sha is not None:
+            return 'user', remote_refname, sha
+
+    # Check upstream project for a branch
+    remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
+    if sha is not None:
+        return 'origin', remote_refname, sha
+
+    # This should never occur given the upstream fallback above
+    print(f"Could not find anything for {module}:{branchname}")
+    print("If something reaches that point, please file a bug")
+    print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues")
+    assert False
+
+
+# --- Unit tests --- #
+# Basically, pytest will happily let a test mutate a variable, and then run
+# the next tests one the same environment without reset the vars.
+def preserve_ci_vars(func):
+    """Preserve the original CI Variable values"""
+    def wrapper():
+        try:
+            url = os.environ["CI_PROJECT_URL"]
+            user = os.environ["CI_PROJECT_NAMESPACE"]
+        except KeyError:
+            url = "invalid"
+            user = ""
+
+        private = os.getenv("READ_PROJECTS_TOKEN", default=None)
+        if not private:
+            os.environ["READ_PROJECTS_TOKEN"] = "FOO"
+
+        func()
+
+        os.environ["CI_PROJECT_URL"] = url
+        os.environ["CI_PROJECT_NAMESPACE"] = user
+
+        if private:
+            os.environ["READ_PROJECTS_TOKEN"] = private
+        # if it was set after
+        elif os.getenv("READ_PROJECTS_TOKEN", default=None):
+            del os.environ["READ_PROJECTS_TOKEN"]
+
+    return wrapper
+
+@preserve_ci_vars
+def test_find_repository_sha():
+    os.environ["CI_PROJECT_NAME"] = "some-random-project"
+    os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
+    os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
+    os.environ["GST_UPSTREAM_BRANCH"] = "master"
+    del os.environ["READ_PROJECTS_TOKEN"]
+
+    # This should find the repository in the user namespace
+    remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2")
+    assert remote == "user"
+    assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325"
+    assert refname == "refs/heads/1.2"
+
+    # This should fallback to upstream master branch since no matching branch was found
+    remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name")
+    assert remote == "origin"
+    assert refname == "refs/heads/master"
+
+    os.environ["CI_PROJECT_NAME"] = "the_project"
+    os.environ["CI_COMMIT_SHA"] = "MySha"
+
+    remote, refname, git_ref = find_repository_sha("the_project", "whatever")
+    assert remote == "user"
+    assert git_ref == "MySha"
+    assert refname == "whatever"
+
+
+@preserve_ci_vars
+def test_get_project_branch():
+    os.environ["CI_PROJECT_NAME"] = "some-random-project"
+    os.environ["CI_COMMIT_SHA"] = "dwbuiw"
+    os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
+    os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
+    del os.environ["READ_PROJECTS_TOKEN"]
+
+    os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
+    remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
+    assert twelve is not None
+    assert remote == 'origin'
+    assert refname == "refs/heads/1.12"
+
+    os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
+    remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
+    assert fourteen is not None
+    assert remote == 'origin'
+    assert refname == "refs/heads/1.14"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--self-update", action="store_true", default=False)
+    parser.add_argument(dest="output", default='manifest.xml', nargs='?')
+    options = parser.parse_args()
+
+    current_branch: str = os.environ['CI_COMMIT_REF_NAME']
+    user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL'])
+    if not user_remote_url.endswith('/'):
+        user_remote_url += '/'
+
+    if options.self_update:
+        remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch)
+        if remote == 'user':
+            remote = user_remote_url + 'gst-ci'
+        else:
+            remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci"
+
+        git('fetch', remote, remote_refname)
+        git('checkout', '--detach', sha)
+        sys.exit(0)
+
+    projects: str = ''
+    for module in GSTREAMER_MODULES:
+        print(f"Checking {module}:", end=' ')
+        remote, refname, revision = find_repository_sha(module, current_branch)
+        print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'")
+        projects += f"  <project path=\"{module}\" name=\"{module}.git\" remote=\"{remote}\" revision=\"{revision}\" refname=\"{refname}\" />\n"
+
+    with open(options.output, mode='w') as manifest:
+        print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest)

ci/gitlab/clone_manifest_ref.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+
+from collections import namedtuple
+import xml.etree.ElementTree as ET
+
+# Disallow git prompting for a username/password
+os.environ['GIT_TERMINAL_PROMPT'] = '0'
+def git(*args, repository_path='.'):
+    return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
+
+class Manifest(object):
+    '''
+    Parse and store the content of a manifest file
+    '''
+
+    remotes = {}
+    projects = {}
+    default_remote = 'origin'
+    default_revision = 'refs/heads/master'
+
+    def __init__(self, manifest_path):
+        self.manifest_path = manifest_path
+
+    def parse(self):
+        try:
+            tree = ET.parse(self.manifest_path)
+        except Exception as ex:
+            raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex))
+
+        root = tree.getroot()
+
+        for child in root:
+            if child.tag == 'remote':
+                self.remotes[child.attrib['name']] = child.attrib['fetch']
+            if child.tag == 'default':
+                self.default_remote = child.attrib['remote'] or self.default_remote
+                self.default_revision = child.attrib['revision'] or self.default_revision
+            if child.tag == 'project':
+                project = namedtuple('Project', ['name', 'remote',
+                    'revision', 'fetch_uri'])
+
+                project.name = child.attrib['name']
+                if project.name.endswith('.git'):
+                    project.name = project.name[:-4]
+                project.remote = child.attrib.get('remote') or self.default_remote
+                project.revision = child.attrib.get('revision') or self.default_revision
+                project.fetch_uri = self.remotes[project.remote] + project.name + '.git'
+
+                self.projects[project.name] = project
+
+    def find_project(self, name):
+        try:
+            return self.projects[name]
+        except KeyError as ex:
+            raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path))
+
+    def get_fetch_uri(self, project, remote):
+        fetch = self.remotes[remote]
+        return fetch + project.name + '.git'
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--project", action="store", type=str)
+    parser.add_argument("--destination", action="store", type=str, default='.')
+    parser.add_argument("--manifest", action="store", type=str)
+    parser.add_argument("--fetch", action="store_true", default=False)
+    options = parser.parse_args()
+
+    if not options.project:
+        raise ValueError("--project argument not provided")
+    if not options.manifest:
+        raise ValueError("--manifest argument not provided")
+
+    manifest = Manifest(options.manifest)
+    manifest.parse()
+    project = manifest.find_project(options.project)
+
+    dest = options.destination
+    if dest == '.':
+        dest = os.path.join (os.getcwd(), project.name)
+
+    if options.fetch:
+        assert os.path.exists(dest) == True
+        git('fetch', project.fetch_uri, project.revision, repository_path=dest)
+    else:
+        git('clone', project.fetch_uri, dest)
+
+    git('checkout', '--detach', project.revision, repository_path=dest)

ci/gitlab/freedesktop_doc_importer.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+import os
+import gitlab
+from datetime import datetime
+import tempfile
+from subprocess import check_call, call, check_output
+
+BRANCH="main"
+NAMESPACE="gstreamer"
+JOB="documentation"
+DOC_BASE="/srv/gstreamer.freedesktop.org/public_html/documentation"
+
+print(f"Running at {datetime.now()}")
+with tempfile.TemporaryDirectory() as tmpdir:
+    os.chdir(tmpdir)
+
+    gl = gitlab.Gitlab("https://gitlab.freedesktop.org/")
+    project = gl.projects.get(1357)
+    pipelines = project.pipelines.list()
+    for pipeline in pipelines:
+        if pipeline.ref != BRANCH:
+            continue
+
+        job, = [j for j in pipeline.jobs.list() if j.name == "documentation"]
+        if job.status != "success":
+            continue
+
+        url = f"https://gitlab.freedesktop.org/gstreamer/gstreamer/-/jobs/{job.id}/artifacts/download"
+        print("============================================================================================================================")
+        print(f"Updating documentation from: {url}\n\n")
+        check_call(f"wget {url} -O gstdocs.zip", shell=True)
+        print("Unziping file.")
+        check_output("unzip gstdocs.zip", shell=True)
+        print("Running rsync.")
+        call(f"rsync -rvaz --links --delete documentation/ {DOC_BASE}", shell=True)
+        call(f"chmod -R g+w {DOC_BASE}; chgrp -R gstreamer {DOC_BASE}", shell=True)
+
+        print(f"Done updating doc")
+        break

ci/gitlab/trigger_cerbero_pipeline.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python3
+
+import time
+import os
+import sys
+import gitlab
+
+CERBERO_PROJECT = 'gstreamer/cerbero'
+
+
+
+class Status:
+    FAILED = 'failed'
+    MANUAL = 'manual'
+    CANCELED = 'canceled'
+    SUCCESS = 'success'
+    SKIPPED = 'skipped'
+    CREATED = 'created'
+
+    @classmethod
+    def is_finished(cls, state):
+        return state in [
+            cls.FAILED,
+            cls.MANUAL,
+            cls.CANCELED,
+            cls.SUCCESS,
+            cls.SKIPPED,
+        ]
+
+
+def fprint(msg):
+    print(msg, end="")
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    server = os.environ['CI_SERVER_URL']
+    gl = gitlab.Gitlab(server,
+        private_token=os.environ.get('GITLAB_API_TOKEN'),
+        job_token=os.environ.get('CI_JOB_TOKEN'))
+
+    cerbero = gl.projects.get(CERBERO_PROJECT)
+    pipe = cerbero.trigger_pipeline(
+        token=os.environ['CI_JOB_TOKEN'],
+        ref=os.environ["GST_UPSTREAM_BRANCH"],
+        variables={
+            "CI_GSTREAMER_URL": os.environ["CI_PROJECT_URL"],
+            "CI_GSTREAMER_REF_NAME": os.environ["CI_COMMIT_REF_NAME"],
+            # This tells cerbero CI that this is a pipeline started via the
+            # trigger API, which means it can use a deps cache instead of
+            # building from scratch.
+            "CI_GSTREAMER_TRIGGERED": "true",
+        }
+    )
+
+    fprint(f'Cerbero pipeline running at {pipe.web_url} ')
+    while True:
+        time.sleep(15)
+        pipe.refresh()
+        if Status.is_finished(pipe.status):
+            fprint(f": {pipe.status}\n")
+            sys.exit(0 if pipe.status == Status.SUCCESS else 1)
+        else:
+            fprint(".")