summaryrefslogtreecommitdiffstats
path: root/roles/bodhi2/backend/templates/owner-sync-pagure.j2
blob: 2b634d0dfa125c4c64c95db2316ccae0809a512c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
#!/usr/bin/python2
"""
This script updates the packageList ownership in Koji based on repo ownership
in Pagure.

This is heavily based on "owner-sync-pkgdb.j2" which was introduced in commit
8c2130080c by Luke Macken.
"""

# cronjobs are run on releng01.stg
# Looks like:
# /usr/local/bin/owner-sync-pkgdb f19 
# /usr/local/bin/owner-sync-pkgdb dist-5E-epel 
# /usr/local/bin/owner-sync-pkgdb dist-6E-epel
# /usr/local/bin/owner-sync-pkgdb epel7 
from __future__ import print_function
import sys
import os
import argparse
import ConfigParser
from urlparse import urljoin
import multiprocessing.pool
from math import ceil
from functools import partial

import requests
import koji

from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry


def retry_session():
    session = requests.Session()
    retry = Retry(
        total=5,
        read=5,
        connect=5,
        backoff_factor=0.3,
        status_forcelist=(500, 502, 504),
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session


# Ansible configured global variables
STAGING = {{ 'True' if env == 'staging' else 'False' }}
HOSTNAME = 'bodhi{{ env_suffix }}.fedoraproject.org'
IPA_REALM = '{{ ipa_realm }}'
ENV_SUFFIX = '{{ env_suffix }}'
if STAGING:
    PAGURE_URL = 'https://src.stg.fedoraproject.org/'
    PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
else:
    PAGURE_URL = 'https://src.fedoraproject.org/'
    PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1/'
# In case the above variables end up being filled in by Ansible
if not PAGURE_URL.endswith('/'):
    PAGURE_URL = PAGURE_URL + '/'
if not PDC_URL.endswith('/'):
    PDC_URL = PDC_URL + '/'

PDC_TYPES = {
    'rpms': 'rpm',
    'modules': 'module',
    'flatpaks': 'flatpak',
    'container': 'container',
}


RAWHIDE = '{{ FedoraRawhideNumber }}'
EXTRA_ARCH_LIST = {
    'kernel': ('i586', 'i686', 'noarch'),
    'kernel-xen-2.6': ('i586', 'i686', 'noarch'),
    'glibc': ('i686',),
    'openssl': ('i686',),
    'em8300-kmod': ('i586', 'i686'),
    'sysprof-kmod': ('i586', 'i686'),
}
VERIFY = True


def usage():
    print('Usage: owner-sync <tag>\n\t<tag>: tag to synchronize owners for',
          file=sys.stderr)
    sys.exit(1)


def get_options():
    # shamelessly stolen from koji CLI
    if STAGING:
        opts = {
            'server': 'https://koji.stg.fedoraproject.org/kojihub',
            'weburl': 'https://koji.stg.fedoraproject.org/koji',
        }
    else:
        opts = {
            'server': 'https://koji.fedoraproject.org/kojihub',
            'weburl': 'https://koji.fedoraproject.org/koji',
        }
    opts['principal'] = 'bodhi/{0}@{1}'.format(HOSTNAME, IPA_REALM)
    opts['keytab'] = '/etc/krb5.bodhi_{0}.keytab'.format(HOSTNAME)

    for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')):
        if os.access(configFile, os.F_OK):
            f = open(configFile)
            config = ConfigParser.ConfigParser()
            config.readfp(f)
            f.close()
            if config.has_section('koji'):
                for name, value in config.items('koji'):
                    if opts.has_key(name):
                        opts[name] = value
    for entry in opts.keys():
        if entry == 'server' or entry == 'weburl':
            pass
    return opts


def get_namespaces_and_version_from_tag(tag):
    if 'container' in tag:
        namespaces = ['container']
        version = tag.split('-')[0].split('f')[1]
    elif 'docker' in tag:
        namespaces = ['container']
        version = tag.split('-')[0].split('f')[1]
    elif 'modular' in tag:
        namespaces = ['flatpaks', 'modules']
        try:
            version = tag.split('-')[0].split('f')[1]
        except IndexError:
            version = RAWHIDE
    elif 'flatpak' in tag:
        namespaces = ['flatpaks']
        version = tag.split('-')[0].split('f')[1]
    elif tag == 'module-package-list':
        # See https://pagure.io/releng/issue/6663
        # and https://pagure.io/fm-orchestrator/issue/333
        namespaces = ['rpms']
        version = RAWHIDE
    else:
        namespaces = ['rpms']
        if tag.startswith('epel'):
            version = tag.split('epel')[1]
        elif tag.startswith('f'):
            version = tag.split('f')[1]
        elif tag.endswith('epel') and tag.startswith('dist'):
            # This is for older EPEL tags such as dit-6E-epel
            version = tag.split('-')[1][:-1]
        else:
            print('Error: an invalid tag was specified', file=sys.stderr)
            sys.exit(1)

    return namespaces, version


def get_branch_and_arches(tag, version):
    if tag.startswith('epel'):
        # Ex: epel7 => epel7
        branch = tag
        arches = ["primary"]
    elif tag.endswith('epel'):
        # Ex: dist-6E-epel => el6
        branch = 'el%s' % version
        arches = ["primary"]
    elif tag == 'module-package-list':
        branch = 'master'
        arches = ["primary"]
    else:
        # Fedora
        if version == RAWHIDE:
            branch = 'master'
        else:
            branch = tag.split('-')[0]

        if STAGING:
            arches = ["primary"]
        else:
            if version <= "26":
                arches = ["primary", "s390"]
            else:
                # Yay!  Everything in primary.
                arches = ["primary"]

    return branch, arches


def get_pdc_project_name_and_branch(session, namespace, project_name,
                                    verbose=False):
    """
    Gets the branches on a project. This function is used for mapping.
    :param namespace: string of the namespace the project is in
    :param project: string of the project
    :param verbose: prints out verbose information
    :return: a tuple containing the string of the project and a list of
    branches
    """
    project_branches_url = '{0}component-branches/'.format(PDC_URL)
    params = dict(
        global_component=project_name,
        type=PDC_TYPES[namespace],
        active=True,
    )
    if verbose:
        print('- Querying {0} {1}'.format(project_branches_url, params))
    project_branches_rv = session.get(
        project_branches_url, params=params, verify=VERIFY, timeout=60)

    # If the project's branches can't be reported, just return no branches and
    # it will be skipped later on
    if not project_branches_rv.ok:
        return project_name, []

    data = project_branches_rv.json()
    return project_name, [branch['name'] for branch in data['results']]


def get_pagure_project_names_from_page(session, namespace, page,
                                       package=None, verbose=False):
    """
    Gets the names of all the Pagure projects on a page. This function is to be
    used for mapping.
    :param namespace: string of the namespace to query for projects
    :param page: int of the page to query at
    :param verbose: prints out verbose information
    :return: list of project names on the page
    """
    url = urljoin(PAGURE_URL, 'api/0/projects?namespace={0}'.format(namespace))
    url = url + '&page={0}&per_page=100&fork=false&short=true'.format(page)
    if package:
        url = url + "&pattern={0}".format(package)
    if verbose:
        print('- Querying {0}'.format(url))
    response = session.get(url, verify=VERIFY, timeout=120)
    if not bool(response):
        print("Failed to talk to %r %r." % (
            response.request.url, response), file=sys.stderr)
        return set()

    names = set()
    for project in response.json()['projects']:
        names.add(project['name'])

    return names


def get_pagure_project_branches(namespace, package=None, verbose=False):
    """
    Gets all the branches of all the Pagure projects in the desired namespace
    :param namespace: string of the namespace to query for projects
    :param verbose: prints out verbose information
    :return: dictionary in the format of {project_name: [branch_one...]}
    """
    first_page_url_path = ('api/0/projects?namespace={0}&fork=false&short=true'
                           '&page=1&per_page=1'.format(namespace))
    first_page_url = urljoin(PAGURE_URL, first_page_url_path)
    if package:
        first_page_url = first_page_url + "&pattern={0}".format(package)
    session = retry_session()
    if verbose:
        print('- Querying {0}'.format(first_page_url))
    first_page_rv = session.get(first_page_url, verify=VERIFY, timeout=120)
    if not bool(first_page_rv):
        print("Failed to talk to %r %r." % (
            first_page_rv.request.url, first_page_rv), file=sys.stderr)
        return {}
    total_projects = first_page_rv.json()['total_projects']
    num_pages = int(ceil(total_projects / 100.0))

    pool = multiprocessing.pool.ThreadPool(8)
    # Since we are going to multi-thread, we need to make a partial function
    # call so that all the function needs is an iterable to run
    partial_get_pagure_project_names_from_page = partial(
        get_pagure_project_names_from_page, session, namespace,
        package=package, verbose=verbose)
    project_names_sets = pool.map(partial_get_pagure_project_names_from_page,
                                  range(1, num_pages + 1))

    if project_names_sets:
        # Combine all the project name sets
        project_names = list(set.union(*project_names_sets))
        # Hopefully save some RAM
        del project_names_sets
    else:
        return {}

    # Since we are going to multi-thread, we need to make a partial function
    # call so that all the function needs is an iterable to run
    partial_get_pdc_project_name_and_branch = partial(
        get_pdc_project_name_and_branch, session, namespace,
        verbose=verbose)
    # Get a list of tuples in the form of (project, [branch...]), then convert
    # that to a dictionary
    project_names_to_branches = dict(pool.map(
        partial_get_pdc_project_name_and_branch, project_names))
    pool.close()
    return project_names_to_branches


def set_koji_ownership(tag, packages, arches, verbose=False):
    koji_login_options = get_options()

    koji_options={
        'krb_rdns': False,
        # About ten minutes.  The default is 12 hours.
        'timeout': 60 * 10,
    }
    for arch in arches:

        # Something weird here.  Where do 'arm' and 'ppc' come from?
        if arch in ['arm', 'ppc']:
            arch = 'primary'

        if arch == 'primary':
            session = koji.ClientSession(
                'https://koji{0}.fedoraproject.org/kojihub'.format(ENV_SUFFIX),
                opts=koji_options,
            )
        else:
            session = koji.ClientSession(
                'https://{0}.koji.fedoraproject.org/kojihub'.format(arch),
                opts=koji_options,
            )

        try:
            session.krb_login(koji_login_options['principal'], koji_login_options['keytab'])
        except:
            import traceback
            traceback.print_exc()
            print('Failed to login to "{0}" hub'.format(arch), file=sys.stderr)
            continue

        kojitag = session.getTag(tag)
        if kojitag is None:
            print('Error: tag "{0}" does not exist for arch "{1}"'
                  .format(tag, arch), file=sys.stderr)
            continue

        koji_pkgs = {}

        if verbose:
            print('- Getting a list of packages in Koji from the tag "{0}" '
                  'and arch "{1}"'.format(tag, arch))

        for p in session.listPackages(tagID=tag, inherited=True):
            koji_pkgs[p['package_name']] = p

        for pkg in packages:
            owner = 'releng'  # always
            if pkg not in koji_pkgs:
                extra_arches = None
                if pkg in EXTRA_ARCH_LIST:
                    extra_arches = EXTRA_ARCH_LIST[pkg]
                if verbose:
                    print('- Adding the package "{0}" to the package list for '
                          'the tag "{1}" on arch "{2}" and applicable extra '
                          'arches'.format(pkg, tag, arch))
                session.packageListAdd(
                    tag, pkg, owner=owner, extra_arches=extra_arches)
            elif koji_pkgs[pkg]['owner_name'] != owner:
                if verbose:
                    print('- Setting the owner on package "{0}" for the tag '
                          '"{1}" on arch "{2}"'
                          .format(pkg, tag, arch))
                session.packageListSetOwner(tag, pkg, owner, force=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser.add_argument('tag', nargs='+',
                        help='tag to update the package list on')
    parser.add_argument('--package', help='Name of an optional single package to sync.')
    parser.add_argument('--verbose', action='store_true')
    args = parser.parse_args()
    verbose = args.verbose
    tags = args.tag
    package = args.package

    # Get all the info about the tags we are interested in
    unique_namespaces = set()
    tag_info = {}
    for tag in tags:
        namespaces, version = get_namespaces_and_version_from_tag(tag)
        branch, arches = get_branch_and_arches(tag, version)
        tag_info[tag] = {
            'namespaces': namespaces,
            'version': version,
            'branch': branch,
            'arches': arches
        }
        unique_namespaces.update(namespaces)

    # Get all the project to branch mappings for every namespace
    namespace_to_projects = {}
    for namespace in unique_namespaces:
        if verbose:
            print('Querying for all the projects with the namespace "{0}"'
                  .format(namespace))
        namespace_to_projects[namespace] = \
            get_pagure_project_branches(namespace, package=package, verbose=verbose)

    for tag, info in tag_info.items():
        if verbose:
            print('Determining which projects have the namespace "{0}" and '
                  'branch "{1}"'.format(namespace, tag))
        namespaces = info['namespaces']
        pkgs = []
        for namespace in namespaces:
            for pkg, branches in namespace_to_projects[namespace].items():
                if info['branch'] in branches or tag == ('f' + RAWHIDE):
                    # The tag and branch names are the same for "old-style" branches
                    pkgs.append(pkg)
                elif namespace in ('modules', 'flatpaks'):
                    # Add modules to f27-modular-updates even if their only branch is '2.4'
                    # Similarly, flatpaks will be built into f29-flatpak-updates-candidate
                    # if they use the f29 runtime, even from master or stream branches.
                    pkgs.append(pkg)

            # This is a special project, not in dist-git, but which needs to be in
            # the package list.
            if namespace == 'rpms':
                pkgs.append('module-build-macros')

        if verbose:
            print('Setting the Koji ownership and package list on packages in '
                  'the tag "{0}" and namespaces "{1}" and for arches "{2}"'
                  .format(tag, namespaces, ', '.join(info['arches'])))
        set_koji_ownership(tag, pkgs, info['arches'], verbose=verbose)