Generate plugin list for registry document

This commit makes to generate a plugin list for the registry
document and, also fixes a python3 incompatibility and docs warnings
in generate-grenade-plugins-list.sh.
And this commit also removes doc/source/plugin-registry.rst because
it's generated by the script.

Change-Id: I391ca452f2e99e899ded8e4bcb5649a4511a7696

.gitignore

@@ -3,6 +3,7 @@ localrc
 *.swp
 doc/files
 doc/build
+doc/source/plugin-registry.rst
 shocco
 .tox
 AUTHORS

data/grenade-plugins-registry.header

@@ -19,6 +19,6 @@ The following are plugins that a script has found in the openstack/
 namespace, which includes but is not limited to official OpenStack
 projects.
 
-+----------------------------+-------------------------------------------------------------------------+
++----------------------------------------+-------------------------------------------------------------------------+
-|Plugin Name                 |URL                                                                      |
+|Plugin Name                             |URL                                                                      |
-+----------------------------+-------------------------------------------------------------------------+
++----------------------------------------+-------------------------------------------------------------------------+

doc/source/conf.py

@@ -11,8 +11,18 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
 import os
+import subprocess
+
+# Build the plugin registry
+def build_plugin_registry(app):
+    root_dir = os.path.dirname(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    subprocess.call(['tools/generate-grenade-plugins-list.sh'], cwd=root_dir)
+
+def setup(app):
+    if os.getenv('GENERATE_GRENADE_PLUGIN_LIST', 'true').lower() == 'true':
+        app.connect('builder-inited', build_plugin_registry)
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the

doc/source/plugin-registry.rst

@@ -1,24 +0,0 @@
-..
-  Note to patch submitters: this file is about to be covered by a
-  periodic proposal job.  You should edit the files
-  data/grenade-plugins-registry.footer and
-  data/grenade-plugins-registry.header instead of this one.
-
-=========================
- Grenade Plugin Registry
-=========================
-
-Since we've created the external plugin mechanism, it's gotten used by
-a lot of projects. The following will be a list of plugins that currently
-exist.
-
-Detected Plugins
-================
-
-The following are plugins that a script has found in the openstack/
-namespace, which includes but is not limited to official OpenStack
-projects.
-
-+----------------------------+-------------------------------------------------------------------------+
-|Plugin Name                 |URL                                                                      |
-+----------------------------+-------------------------------------------------------------------------+

tools/generate-grenade-plugins-list.py

@@ -24,7 +24,14 @@
 #   * network access to https://git.openstack.org/cgit
 
 import json
-import requests
+try:
+    # For Python 3.0 and later
+    from urllib.error import HTTPError
+    import urllib.request as urllib
+except ImportError:
+    # Fall back to Python 2's urllib2
+    import urllib2 as urllib
+    from urllib2 import HTTPError
 
 url = 'https://review.openstack.org/projects/'
 
@@ -39,21 +46,20 @@ url = 'https://review.openstack.org/projects/'
 def is_in_openstack_namespace(proj):
     return proj.startswith('openstack/')
 
-# Rather than returning a 404 for a nonexistent file, cgit delivers a
-# 0-byte response to a GET request.  It also does not provide a
-# Content-Length in a HEAD response, so the way we tell if a file exists
-# is to check the length of the entire GET response body.
-def has_grenade_plugin(proj):
-    r = requests.get("https://git.openstack.org/cgit/%s/plain/devstack/upgrade/upgrade.sh" % proj)
-    if len(r.text) > 0:
-        return True
-    else:
-        False
 
-r = requests.get(url)
+def has_grenade_plugin(proj):
-projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
+    try:
+        r = urllib.urlopen(
+            "https://git.openstack.org/cgit/%s/plain/devstack/upgrade/upgrade.sh" % proj)
+        return True
+    except HTTPError as err:
+        if err.code == 404:
+            return False
+
+r = urllib.urlopen(url)
+projects = sorted(filter(is_in_openstack_namespace, json.loads(r.read()[4:])))
 
 found_plugins = filter(has_grenade_plugin, projects)
 
 for project in found_plugins:
-    print project[10:]
+    print(project[10:])

tools/generate-grenade-plugins-list.sh

@@ -48,10 +48,10 @@ fi
 sorted_plugins=$(python tools/generate-grenade-plugins-list.py)
 
 for k in ${sorted_plugins}; do
-    project=${k:0:28}
+    project=${k:0:40}
-    giturl="git://git.openstack.org/openstack/${k:0:26}"
+    giturl="git://git.openstack.org/openstack/${k:0:36}"
-    printf "|%-28s|%-73s|\n" "${project}" "${giturl}"
+    printf "|%-40s|%-73s|\n" "${project}" "${giturl}"
-    printf "+----------------------------+-------------------------------------------------------------------------+\n"
+    printf "+----------------------------------------+-------------------------------------------------------------------------+\n"
 done
 
 if [[ -r data/grenade-plugins-registry.footer ]]; then

tox.ini

@@ -7,6 +7,7 @@ skipsdist = True
 install_command = pip install -U {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
+passenv = GENERATE_GRENADE_PLUGIN_LIST
 
 [testenv:venv]
 commands = {posargs}