From 1660e2757ab8b02f880d56796ac539058b57520d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 1 Jun 2012 11:46:54 -0400 Subject: [PATCH] Add first stab at pip-cache to pypi-mirror. Change-Id: I54a671ebddd523d9d22eec5e0a3f03fae8faa4f1 --- modules/pypimirror/files/process_cache.py | 89 +++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 modules/pypimirror/files/process_cache.py diff --git a/modules/pypimirror/files/process_cache.py b/modules/pypimirror/files/process_cache.py new file mode 100644 index 0000000000..938068e9fb --- /dev/null +++ b/modules/pypimirror/files/process_cache.py @@ -0,0 +1,89 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import urllib +import datetime +import sys +import re +import md5 + +source_cache = sys.argv[1] +destination_mirror = sys.argv[2] + +PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]') + +packages = {} +package_count = 0 + +for filename in os.listdir(source_cache): + if filename.endswith('content-type'): + continue + + realname = urllib.unquote(filename) + tarball = os.path.basename(realname) + name_match = PACKAGE_VERSION_RE.search(tarball) + + if name_match is None: + continue + package_name = name_match.group(1) + + version_list = packages.get(package_name,{}) + version_list[tarball] = filename + packages[package_name] = version_list + package_count = package_count + 1 + +full_html = open(os.path.join(destination_mirror, "full.html"), 'w') +simple_html = open(os.path.join(destination_mirror, "index.html"), 'w') + +header = "PyPI Mirror

PyPI Mirror

Last update: %s

\n\n" % datetime.datetime.utcnow().strftime("%c UTC") +full_html.write(header) +simple_html.write(header) + +for package_name, versions in packages.items(): + destination_dir = os.path.join(destination_mirror, package_name) + if not os.path.isdir(destination_dir): + os.makedirs(destination_dir) + safe_dir = urllib.quote(package_name) + simple_html.write("%s
\n" % (safe_dir, safe_dir)) + with open(os.path.join(destination_dir, "index.html"), 'w') as index: + index.write(""" + %s – PyPI Mirror +\n""" % package_name) + for tarball, filename in versions.items(): + source_path = os.path.join(source_cache, filename) + destination_path = os.path.join(destination_dir, tarball) + with open(destination_path, 'w') as dest: + src = open(source_path, 'r').read() + md5sum = md5.md5(src).hexdigest() + dest.write(src) + + safe_name = urllib.quote(tarball) + + full_html.write("%s
\n" % (safe_dir, + safe_name, + safe_name)) + index.write("%s\n" % (safe_name, + md5sum, + safe_name)) + index.write("\n") +footer = """

Generated by process_cache.py; %d +packages mirrored.

+\n""" % package_count +full_html.write(footer) +full_html.close() +simple_html.write(footer) +simple_html.close()