Initial checkin of ATC scripts
Change-Id: Ibffab01bc835d380ca9f0c18521c8a6d808de761
This commit is contained in:
James E. Blair
parent
43bad50bea
commit
96f2f6ddd9
33
tools/atc/README
Normal file
33
tools/atc/README
Normal file
@ -0,0 +1,33 @@
|
||||
These are the scripts used to create the ATC lists for use in PTL
|
||||
elections and Summit invitations.
|
||||
|
||||
0) Write a patch to email-stats.py so steps 1-2 are not necessary.
|
||||
1) Edit email-stats.py to set your gerrit username.
|
||||
2) Edit email-stats.py to set the start_date and end_date.
|
||||
3) Run the following queries on review.openstack.org:
|
||||
|
||||
SELECT * FROM accounts
|
||||
INTO OUTFILE '/tmp/accounts.csv'
|
||||
FIELDS TERMINATED BY ','
|
||||
ENCLOSED BY '"'
|
||||
LINES TERMINATED BY '\n';
|
||||
|
||||
SELECT * FROM account_external_ids
|
||||
INTO OUTFILE '/tmp/emails.csv'
|
||||
FIELDS TERMINATED BY ','
|
||||
ENCLOSED BY '"'
|
||||
LINES TERMINATED BY '\n';
|
||||
|
||||
4) Copy those files to this directory.
|
||||
5) Run:
|
||||
|
||||
mkdir out
|
||||
./email-stats.sh
|
||||
DATE=`date --iso`
|
||||
mkdir $DATE
|
||||
cat out/*.csv | sort | uniq > $DATE/all.csv
|
||||
mv out/* $DATE/
|
||||
|
||||
6) You can use diff.py to get the new ATCs since the previous run:
|
||||
|
||||
./diff.py $OLD-DATE/all.csv $DATE/all.csv $DATE/new.csv
|
||||
16
tools/atc/diff.py
Executable file
16
tools/atc/diff.py
Executable file
@ -0,0 +1,16 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import csv
|
||||
import sys
|
||||
|
||||
old = {}
|
||||
new = {}
|
||||
|
||||
for row in csv.reader(open(sys.argv[1])):
|
||||
old[row[0]] = row
|
||||
|
||||
writer = csv.writer(open(sys.argv[3], 'w'))
|
||||
for row in csv.reader(open(sys.argv[2])):
|
||||
if row[0] not in old:
|
||||
writer.writerow(row)
|
||||
|
||||
141
tools/atc/email-stats.py
Executable file
141
tools/atc/email-stats.py
Executable file
@ -0,0 +1,141 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
# Copyright (C) 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Soren Hansen wrote the original version of this script.
|
||||
# James Blair hacked it up to include email addresses from gerrit.
|
||||
|
||||
import calendar
|
||||
import datetime
|
||||
import json
|
||||
import optparse
|
||||
import paramiko
|
||||
from pprint import pprint
|
||||
import sys
|
||||
import csv
|
||||
import re
|
||||
|
||||
MAILTO_RE = re.compile('mailto:(.*)')
|
||||
USERNAME_RE = re.compile('username:(.*)')
|
||||
accounts = {}
|
||||
|
||||
class Account(object):
|
||||
def __init__(self, num):
|
||||
self.num = num
|
||||
self.full_name = ''
|
||||
self.emails = []
|
||||
self.username = None
|
||||
|
||||
def get_account(num):
|
||||
a = accounts.get(num)
|
||||
if not a:
|
||||
a = Account(num)
|
||||
accounts[num] = a
|
||||
return a
|
||||
|
||||
for row in csv.reader(open('emails.csv')):
|
||||
num, email, pw, external = row
|
||||
num = int(num)
|
||||
a = get_account(num)
|
||||
if email and email != '\\N' and email not in a.emails:
|
||||
a.emails.append(email)
|
||||
m = MAILTO_RE.match(external)
|
||||
if m:
|
||||
if m.group(1) not in a.emails:
|
||||
a.emails.append(m.group(1))
|
||||
m = USERNAME_RE.match(external)
|
||||
if m:
|
||||
if a.username:
|
||||
print a.num
|
||||
print a.username
|
||||
raise Exception("Already a username")
|
||||
a.username = m.group(1)
|
||||
|
||||
for row in csv.reader(open('accounts.csv')):
|
||||
num = int(row[-1])
|
||||
name = row[1]
|
||||
a = get_account(num)
|
||||
a.full_name = name
|
||||
|
||||
username_accounts = {}
|
||||
for a in accounts.values():
|
||||
username_accounts[a.username] = a
|
||||
|
||||
atcs = []
|
||||
|
||||
optparser = optparse.OptionParser()
|
||||
optparser.add_option('-p', '--project', default='nova', help='Project to generate stats for')
|
||||
optparser.add_option('-o', '--output', default='out.csv', help='Output file')
|
||||
options, args = optparser.parse_args()
|
||||
|
||||
QUERY = "project:%s status:merged" % options.project
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
client.load_system_host_keys()
|
||||
client.connect('review.openstack.org', port=29418, key_filename='/home/corvus/.ssh/id_rsa', username='CHANGME')
|
||||
stdin, stdout, stderr = client.exec_command('gerrit query %s --all-approvals --format JSON' %
|
||||
QUERY)
|
||||
changes = []
|
||||
|
||||
done = False
|
||||
last_sortkey = ''
|
||||
tz = datetime.tzinfo
|
||||
start_date = datetime.datetime(2012, 9, 27, 0, 0, 0)
|
||||
end_date = datetime.datetime(2013, 7, 30, 0, 0, 0)
|
||||
|
||||
count = 0
|
||||
earliest = datetime.datetime.now()
|
||||
while not done:
|
||||
for l in stdout:
|
||||
data = json.loads(l)
|
||||
if 'rowCount' in data:
|
||||
if data['rowCount'] < 500:
|
||||
done = True
|
||||
continue
|
||||
count += 1
|
||||
last_sortkey = data['sortKey']
|
||||
if 'owner' not in data:
|
||||
continue
|
||||
if 'username' not in data['owner']:
|
||||
continue
|
||||
account = username_accounts[data['owner']['username']]
|
||||
approved = False
|
||||
for ps in data['patchSets']:
|
||||
if 'approvals' not in ps:
|
||||
continue
|
||||
for aprv in ps['approvals']:
|
||||
if aprv['type'] != 'SUBM':
|
||||
continue
|
||||
ts = datetime.datetime.fromtimestamp(aprv['grantedOn'])
|
||||
if ts < start_date or ts > end_date:
|
||||
continue
|
||||
approved = True
|
||||
if ts < earliest:
|
||||
earliest = ts
|
||||
if approved and account not in atcs:
|
||||
atcs.append(account)
|
||||
if not done:
|
||||
stdin, stdout, stderr = client.exec_command('gerrit query %s resume_sortkey:%s --all-approvals --format JSON' % (QUERY, last_sortkey))
|
||||
|
||||
print 'project: %s' % options.project
|
||||
print 'examined %s changes' % count
|
||||
print 'earliest timestamp: %s' % earliest
|
||||
writer = csv.writer(open(options.output, 'w'))
|
||||
for a in atcs:
|
||||
writer.writerow([a.username, a.full_name] + a.emails)
|
||||
print
|
||||
81
tools/atc/email-stats.sh
Executable file
81
tools/atc/email-stats.sh
Executable file
@ -0,0 +1,81 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Integrated projects
|
||||
python email-stats.py -p openstack/nova -o out/nova.csv
|
||||
python email-stats.py -p openstack/swift -o out/swift.csv
|
||||
python email-stats.py -p openstack/glance -o out/glance.csv
|
||||
python email-stats.py -p openstack/keystone -o out/keystone.csv
|
||||
python email-stats.py -p openstack/horizon -o out/horizon.csv
|
||||
python email-stats.py -p openstack/neutron -o out/neutron.csv
|
||||
python email-stats.py -p openstack/cinder -o out/cinder.csv
|
||||
python email-stats.py -p openstack/ceilometer -o out/ceilometer.csv
|
||||
python email-stats.py -p openstack/heat -o out/heat.csv
|
||||
|
||||
# Library projects
|
||||
python email-stats.py -p openstack/oslo-incubator -o out/olso-incubator.csv
|
||||
python email-stats.py -p openstack/oslo.config -o out/olso.config.csv
|
||||
python email-stats.py -p openstack/python-novaclient -o out/python-novaclient.csv
|
||||
python email-stats.py -p openstack/python-swiftclient -o out/python-swiftclient.csv
|
||||
python email-stats.py -p openstack/python-glanceclient -o out/python-glanceclient.csv
|
||||
python email-stats.py -p openstack/python-keystoneclient -o out/python-keystoneclient.csv
|
||||
python email-stats.py -p openstack/python-neutronclient -o out/python-neutronclient.csv
|
||||
python email-stats.py -p openstack/python-cinderclient -o out/python-cinderclient.csv
|
||||
python email-stats.py -p openstack/python-ceilometerclient -o out/python-ceilometerclient.csv
|
||||
python email-stats.py -p openstack/python-heatclient -o out/python-heatclient.csv
|
||||
python email-stats.py -p openstack/heat-cfntools -o out/heat-cfntools.csv
|
||||
python email-stats.py -p openstack/heat-templates -o out/heat-templates.csv
|
||||
|
||||
# Incubated projects
|
||||
python email-stats.py -p openstack/trove -o out/trove.csv
|
||||
python email-stats.py -p openstack/trove-integration -o out/trove-integration.csv
|
||||
python email-stats.py -p openstack/python-troveclient -o out/python-troveclient.csv
|
||||
python email-stats.py -p openstack/ironic -o out/ironic.csv
|
||||
python email-stats.py -p openstack/python-ironicclient -o out/python-ironicclient.csv
|
||||
|
||||
# Gating projects
|
||||
python email-stats.py -p openstack-dev/devstack -o out/devstack.csv
|
||||
python email-stats.py -p openstack-dev/grenade -o out/grenade.csv
|
||||
python email-stats.py -p openstack-dev/hacking -o out/hacking.csv
|
||||
python email-stats.py -p openstack-dev/pbr -o out/pbr.csv
|
||||
python email-stats.py -p openstack/tempest -o out/tempest.csv
|
||||
python email-stats.py -p openstack-dev/openstack-nose -o out/openstack-nose.csv
|
||||
python email-stats.py -p openstack/requirements -o out/requirements.csv
|
||||
|
||||
# Supporting projects
|
||||
python email-stats.py -p openstack/compute-api -o out/compute-api.csv
|
||||
python email-stats.py -p openstack/identity-api -o out/identity-api.csv
|
||||
python email-stats.py -p openstack/image-api -o out/image-api.csv
|
||||
python email-stats.py -p openstack/netconn-api -o out/netconn-api.csv
|
||||
python email-stats.py -p openstack/object-api -o out/object-api.csv
|
||||
python email-stats.py -p openstack/volume-api -o out/volume-api.csv
|
||||
python email-stats.py -p openstack/openstack-manuals -o out/openstack-manuals.csv
|
||||
python email-stats.py -p openstack/api-site -o out/api-site.csv
|
||||
python email-stats.py -p openstack/operations-guide -o out/operations-guide.csv
|
||||
|
||||
# Infrastructure projects
|
||||
python email-stats.py -p openstack-infra/askbot-theme -o out/askbot-theme.csv
|
||||
python email-stats.py -p openstack-infra/config -o out/config.csv
|
||||
python email-stats.py -p openstack-infra/devstack-gate -o out/devstack-gate.csv
|
||||
python email-stats.py -p openstack-infra/gear -o out/gear.csv
|
||||
python email-stats.py -p openstack-infra/gearman-plugin -o out/gearman-plugin.csv
|
||||
python email-stats.py -p openstack-infra/gerrit -o out/gerrit.csv
|
||||
python email-stats.py -p openstack-infra/gerritbot -o out/gerritbot.csv
|
||||
python email-stats.py -p openstack-infra/gerritlib -o out/gerritlib.csv
|
||||
python email-stats.py -p openstack-infra/git-review -o out/git-review.csv
|
||||
python email-stats.py -p openstack-infra/gitdm -o out/gitdm.csv
|
||||
python email-stats.py -p openstack-infra/jeepyb -o out/jeepyb.csv
|
||||
python email-stats.py -p openstack-infra/jenkins-job-builder -o out/jenkins-job-builder.csv
|
||||
python email-stats.py -p openstack-infra/lodgeit -o out/lodgeit.csv
|
||||
python email-stats.py -p openstack-infra/meetbot -o out/meetbot.csv
|
||||
python email-stats.py -p openstack-infra/nose-html-output -o out/nose-html-output.csv
|
||||
python email-stats.py -p openstack-infra/odsreg -o out/odsreg.csv
|
||||
python email-stats.py -p openstack-infra/publications -o out/publications.csv
|
||||
python email-stats.py -p openstack-infra/puppet-apparmor -o out/puppet-apparmor.csv
|
||||
python email-stats.py -p openstack-infra/puppet-dashboard -o out/puppet-dashboard.csv
|
||||
python email-stats.py -p openstack-infra/puppet-vcsrepo -o out/puppet-vcsrepo.csv
|
||||
python email-stats.py -p openstack-infra/pypi-mirror -o out/pypi-mirror.csv
|
||||
python email-stats.py -p openstack-infra/releasestatus -o out/releasestatus.csv
|
||||
python email-stats.py -p openstack-infra/reviewday -o out/reviewday.csv
|
||||
python email-stats.py -p openstack-infra/statusbot -o out/statusbot.csv
|
||||
python email-stats.py -p openstack-infra/zmq-event-publisher -o out/zmq-event-publisher.csv
|
||||
python email-stats.py -p openstack-infra/zuul -o out/zuul.csv
|
||||
Loading…
x
Reference in New Issue
Block a user