Implemented metadata for services and filesystem, new filesystem snapshot format

config_samples/config/host1/filesystem

@@ -0,0 +1,20 @@
+# type | source filesystem | reference file | owner | group | permissions
+
+dir|/etc/cinder|cinder|cinder|0750
+file|/etc/cinder/cinder.conf|etc/cinder/cinder.conf|cinder|cinder|0640
+file|/etc/cinder/logging.conf|etc/cinder/logging.conf|cinder|cinder|0640
+file|/etc/cinder/api-paste.ini|etc/cinder/api-paste.ini|cinder|cinder|0640
+file|/etc/cinder/policy.json|etc/cinder/policy.json|cinder|cinder|0640
+file|/etc/cinder/rootwrap.conf|etc/cinder/rootwrap.conf|cinder|cinder|0640
+
+dir|/etc/glance|glance|glance|0750
+file|/etc/glance/glance-cache.conf|etc/glance/glance-cache.conf|glance|glance|0640
+file|/etc/glance/glance-api.conf|etc/glance/glance-api.conf|glance|glance|0640
+file|/etc/glance/glance-api-paste.ini|etc/glance/glance-api-paste.ini|glance|glance|0640
+file|/etc/glance/logging.conf|etc/glance/logging.conf|glance|glance|0640
+file|/etc/glance/glance-registry.conf|etc/glance/glance-registry.conf|glance|glance|0640
+file|/etc/glance/policy.json|etc/glance/policy.json|glance|glance|0640
+file|/etc/glance/schema-image.json|etc/glance/schema-image.json|glance|glance|0640
+file|/etc/glance/glance-scrubber.conf|etc/glance/glance-scrubber.conf|glance|glance|0640
+file|/etc/glance/glance-registry-paste.ini|etc/glance/glance-registry-paste.ini|glance|glance|0640
+

config_samples/config/host1/services.json

@@ -0,0 +1,9 @@
+{
+  "cinder": {
+    "version": "2013.1"
+  },
+  "glance": {
+    "version": "2013.1"
+  }
+}
+

config_samples/config/host2/filesystem

@@ -0,0 +1,16 @@
+# type | source filesystem | reference file | owner | group | permissions
+
+dir|/etc/keystone|keystone|keystone|0750
+file|/etc/keystone/logging.conf|etc/keystone/logging.conf|keystone|keystone|0640
+file|/etc/keystone/default_catalog.templates|etc/keystone/default_catalog.templates|keystone|keystone|0640
+file|/etc/keystone/policy.json|etc/keystone/policy.json|keystone|keystone|0640
+file|/etc/keystone/keystone.conf|etc/keystone/keystone.conf|keystone|keystone|0640
+
+dir|/etc/nova|nova|nova|0750
+file|/etc/nova/logging.conf|etc/nova/logging.conf|nova|nova|0640
+file|/etc/nova/api-paste.ini|etc/nova/api-paste.ini|nova|nova|0640
+file|/etc/nova/policy.json|etc/nova/policy.json|nova|nova|0640
+file|/etc/nova/rootwrap.conf|etc/nova/rootwrap.conf|nova|nova|0640
+file|/etc/nova/release|etc/nova/release|nova|nova|0640
+file|/etc/nova/nova.conf|etc/nova/nova.conf|nova|nova|0640
+

config_samples/config/host2/nova/version

@@ -1 +0,0 @@
-2013.1

config_samples/config/host2/services.json

@@ -0,0 +1,9 @@
+{
+  "keystone": {
+    "version": "2013.1"
+  },
+  "nova": {
+    "version": "2013.1"
+  }
+}
+

openstack-configuration.txt

@@ -158,3 +158,95 @@
   
 (continue from http://docs.openstack.org/grizzly/openstack-compute/install/yum/content/compute-minimum-configuration-settings.html)
 
+  * Ensure user 'nova' exists, group 'nova' exists, user 'nova' belongs to group 'nova'
+  * Ensure that '/etc/nova' has 'nova:nova' owners.
+  * Ensure that '/etc/nova/nova.conf' has 'root:nova' owners and 0640 permissions.
+  
+  * Minimal /etc/nova/nova.conf:
+
+    auth_strategy=keystone
+    network_manager=nova.network.manager.FlatDHCPManager
+    fixed_range=192.168.100.0/24
+    public_interface=eth0
+    flat_interface=eth0
+    flat_network_bridge=br100
+
+  * Sample /etc/nova/nova.conf:
+
+    [DEFAULT]
+    
+    # LOGS/STATE
+    verbose=True
+    logdir=/var/log/nova
+    state_path=/var/lib/nova
+    lock_path=/var/lock/nova
+    rootwrap_config=/etc/nova/rootwrap.conf
+    
+    # SCHEDULER
+    compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
+    
+    # VOLUMES
+    volume_api_class=nova.volume.cinder.API
+    volume_driver=nova.volume.driver.ISCSIDriver
+    volume_group=cinder-volumes
+    volume_name_template=volume-%s
+    iscsi_helper=tgtadm
+    
+    # DATABASE
+    sql_connection=mysql://nova:yourpassword@192.168.206.130/nova
+    
+    # COMPUTE
+    libvirt_type=qemu
+    compute_driver=libvirt.LibvirtDriver
+    instance_name_template=instance-%08x
+    api_paste_config=/etc/nova/api-paste.ini
+    
+    # COMPUTE/APIS: if you have separate configs for separate services
+    # this flag is required for both nova-api and nova-compute
+    allow_resize_to_same_host=True
+    
+    # APIS
+    osapi_compute_extension=nova.api.openstack.compute.contrib.standard_extensions
+    ec2_dmz_host=192.168.206.130
+    s3_host=192.168.206.130
+    enabled_apis=ec2,osapi_compute,metadata
+    
+    # QPID
+    qpid_hostname=192.168.206.130
+    
+    # GLANCE
+    image_service=nova.image.glance.GlanceImageService
+    glance_api_servers=192.168.206.130:9292
+    
+    # NETWORK
+    network_manager=nova.network.manager.FlatDHCPManager
+    force_dhcp_release=True
+    dhcpbridge_flagfile=/etc/nova/nova.conf
+    firewall_driver=nova.virt.libvirt.firewall.IptablesFirewallDriver
+    # Change my_ip to match each host
+    my_ip=192.168.206.130
+    public_interface=eth100
+    vlan_interface=eth0
+    flat_network_bridge=br100
+    flat_interface=eth0
+    fixed_range=192.168.100.0/24
+    
+    # NOVNC CONSOLE
+    novncproxy_base_url=http://192.168.206.130:6080/vnc_auto.html
+    # Change vncserver_proxyclient_address and vncserver_listen to match each compute host
+    vncserver_proxyclient_address=192.168.206.130
+    vncserver_listen=192.168.206.130
+    
+    # AUTHENTICATION
+    auth_strategy=keystone
+    [keystone_authtoken]
+    auth_host = 127.0.0.1
+    auth_port = 35357
+    auth_protocol = http
+    admin_tenant_name = service
+    admin_user = nova
+    admin_password = nova
+    signing_dirname = /tmp/keystone-signing-nova
+
+  * 'nova-manage version' to find out version of nova. The output will be something like '2013.1'.
+

ostack_validator/model.py

@@ -1,3 +1,5 @@
+import os.path
+
 from ostack_validator.common import Mark
 
 class Openstack(object):
@@ -19,10 +21,11 @@ class Host(object):
       component.parent = self
 
 class OpenstackComponent(object):
-  def __init__(self, name, version):
+  def __init__(self, name, version, base_path=None):
     super(OpenstackComponent, self).__init__()
     self.name = name
     self.version = version
+    self.base_path = base_path or '/etc/%s' % self.name
     self.configs = {}
 
   @property
@@ -38,7 +41,7 @@ class OpenstackComponent(object):
       config_name = '%s.conf' % self.name
 
     if not config_name in self.configs:
-      resource = self.openstack.resource_locator.find_resource(self.host.name, self.name, config_name)
+      resource = self.openstack.resource_locator.find_resource('file', name=os.path.join(self.base_path, config_name), host=self.host.name)
       if resource:
         config = self.openstack.config_parser.parse(config_name, Mark(resource.name), resource.get_contents())
         self.configs[config_name] = config

ostack_validator/model_parser.py

@@ -2,10 +2,10 @@ import logging
 
 from ostack_validator.common import Version
 from ostack_validator.model import *
-from ostack_validator.resource import ConfigSnapshotResourceLocator
+from ostack_validator.resource import Resource, ConfigSnapshotResourceLocator
 from ostack_validator.config_formats import IniConfigParser
 
-OPENSTACK_COMPONENTS = ['nova', 'keystone', 'glance']
+OPENSTACK_COMPONENTS = ['nova', 'keystone', 'glance', 'cinder', 'horizon', 'quantum', 'swift']
 
 class ModelParser(object):
   logger = logging.getLogger('ostack_validator.ModelParser')
@@ -14,21 +14,15 @@ class ModelParser(object):
     resource_locator = ConfigSnapshotResourceLocator(path)
 
     hosts = []
-    for host_name in resource_locator.find_hosts():
+    for host in resource_locator.find_resource(Resource.HOST):
       components = []
-      for component_name in resource_locator.find_host_components(host_name):
-        if not component_name in OPENSTACK_COMPONENTS:
-          self.logger.warn('Unknown component in config: %s', component_name)
+      for service in host.find_resource(Resource.SERVICE):
+        if not service.name in OPENSTACK_COMPONENTS:
           continue
 
-        component_version = Version(1000000) # very latest version
-        version_resource = resource_locator.find_resource(host_name, component_name, 'version')
-        if version_resource:
-          component_version = Version(version_resource.get_contents())
+        components.append(OpenstackComponent(service.name, service.version))
 
-        components.append(OpenstackComponent(component_name, component_version))
-
-      hosts.append(Host(host_name, {}, components))
+      hosts.append(Host(host.name, {}, components))
 
     return Openstack(hosts, resource_locator, IniConfigParser())
 

ostack_validator/resource.py

@@ -1,60 +1,165 @@
 import glob
 import os.path
+import json
 
-from ostack_validator.common import Error
+from ostack_validator.common import Error, Version
 
 class Resource(object):
+  HOST = 'host'
+  FILE = 'file'
+  DIRECTORY = 'directory'
+  SERVICE = 'service'
+
   def __init__(self, name):
     super(Resource, self).__init__()
     self.name = name
 
+  def __str__(self):
+    return self.name
+
+  def __repr__(self):
+    return '<%s name=%s>' % (str(self.__class__).split('.')[-1], self.name)
+
   def get_contents(self):
     raise Error, 'Not implemented'
 
 class ResourceLocator(object):
-  def find_hosts(self):
-    return []
-
-  def find_host_components(self, host):
-    return []
-
-  def find_resource(self, host, component, name):
+  def find_resource(self, resource_type, name, host=None, **kwargs):
     return None
 
+class HostResource(Resource):
+  def __init__(self, name, resource_locator, interfaces=[]):
+    super(HostResource, self).__init__(name)
+    self.resource_locator = resource_locator
+    self.interfaces = interfaces
+
+  def find_resource(self, resource_type, name=None, **kwargs):
+    return self.resource_locator.find_resource(resource_type, name, host=self, **kwargs)
+
+class DirectoryResource(Resource):
+  def __init__(self, name, owner=None, group=None, permissions=None):
+    super(DirectoryResource, self).__init__(name)
+    self.owner = owner
+    self.group = group
+    self.permissions = permissions
+
 class FileResource(Resource):
-  def __init__(self, name, path):
+  def __init__(self, name, path, owner=None, group=None, permissions=None):
     super(FileResource, self).__init__(name)
     self.path = path
+    self.owner = owner
+    self.group = group
+    self.permissions = permissions
 
   def get_contents(self):
     with open(self.path) as f:
       return f.read()
 
+class ServiceResource(Resource):
+  def __init__(self, name, version, metadata={}):
+    super(ServiceResource, self).__init__(name)
+    self.version = Version(version)
+    self.metadata = metadata
+
+
+class FilesystemSnapshot(object):
+  def __init__(self, path):
+    super(FilesystemSnapshot, self).__init__()
+    self.path = path
+    self.basedir = os.path.join(os.path.dirname(self.path), 'root')
+    self._parse_snapshot()
+
+  def get_resource(self, path):
+    if path in self._resources:
+      return self._resources[path]
+
+    return None
+
+  def _parse_snapshot(self):
+    self._resources = {}
+    if not os.path.isfile(self.path): return
+    with open(self.path) as f:
+      for line in f.readlines():
+        line = line.lstrip()
+        if line == '' or line.startswith('#'): continue
+
+        resource_type = line.split('|')[0]
+        if resource_type == 'dir':
+          source_path, owner, group, permissions = line.split('|')[1:]
+          self._resources[source_path] = DirectoryResource(source_path, owner=owner, group=group, permissions=permissions)
+        elif resource_type == 'file':
+          source_path, local_path, owner, group, permissions = line.split('|')[1:]
+          self._resources[source_path] = FileResource(os.path.basename(source_path), path=os.path.join(self.basedir, local_path), owner=owner, group=group, permissions=permissions)
+        else:
+          self.logger.warn('Unknown resource "%s" in line "%s"' % (resource_type, line))
+
 class ConfigSnapshotResourceLocator(object):
   def __init__(self, basedir):
     super(ConfigSnapshotResourceLocator, self).__init__()
     self.basedir = basedir
     if not os.path.isdir(self.basedir):
       raise Error, 'Invalid argument: base directory does not exist'
+    self._services = None
+    self._filesystem_snapshots = {}
 
-  def find_hosts(self):
-    return [os.path.basename(host_path) for host_path in glob.glob(os.path.join(self.basedir, '*')) if os.path.isdir(host_path)]
+  def find_resource(self, resource_type, name=None, host=None, **kwargs):
+    if resource_type == Resource.HOST:
+      if name:
+        host_path = os.path.join(self.basedir, name)
+        if not os.path.isdir(host_path):
+          return None
+        return HostResource(name, self)
+      else:
+        return [HostResource(os.path.basename(host_path), self) for host_path in glob.glob(os.path.join(self.basedir, '*')) if os.path.isdir(host_path)]
+    if resource_type == Resource.FILE:
+      if not host:
+        raise Error, 'Invalid argument: "host" need to be specified'
 
-  def find_host_components(self, host):
-    return [os.path.basename(component_path) for component_path in glob.glob(os.path.join(self.basedir, host, '*')) if os.path.isdir(component_path)]
+      if isinstance(host, HostResource):
+        host = host.name
 
-  def find_resource(self, host, component, name):
-    if not host:
-      raise Error, 'Invalid argument: "host" need to be specified'
+      if name:
+        return self._get_filesystem_snapshot(host).get_resource(name)
+      else:
+        return []
+    elif resource_type == Resource.SERVICE:
+      if not host:
+        raise Error, 'Invalid argument: "host" need to be specified'
 
-    if not component:
-      raise Error, 'Invalid argument: "component" need to be specified'
+      if isinstance(host, HostResource):
+        host = host.name
 
-    path = os.path.join(self.basedir, host, component, name)
-    if not os.path.exists(path):
+      self._ensure_services_loaded()
+
+      if name:
+        if name in self._services:
+          return self._services[host][name]
+        else:
+          return None
+      else:
+        return self._services[host].values()
+    else:
       return None
 
-    fullname = '%s/%s/%s' % (host, component, name)
+  def _ensure_services_loaded(self):
+    if self._services: return
 
-    return FileResource(fullname, path)
+    self._services = {}
+    for host_path in glob.glob(os.path.join(self.basedir, '*')):
+      if not os.path.isdir(host_path): continue
+
+      services_json_path = os.path.join(host_path, 'services.json')
+      if not os.path.isfile(services_json_path): continue
+
+      host_name = os.path.basename(host_path)
+      self._services[host_name] = {}
+      with open(services_json_path) as f:
+        for service_name, metadata in json.loads(f.read()).items():
+          version = metadata.pop('version')
+          self._services[host_name][service_name] = ServiceResource(service_name, str(version), metadata)
+
+  def _get_filesystem_snapshot(self, host):
+    if not host in self._filesystem_snapshots:
+      self._filesystem_snapshots[host] = FilesystemSnapshot(os.path.join(self.basedir, host, 'filesystem'))
+    return self._filesystem_snapshots[host]