Commit f3d758cd authored by Sylvain's avatar Sylvain
Browse files

schema building refactoring

parent ad6bcf3df1e8
......@@ -2,8 +2,12 @@ ChangeLog for yams
------------------
--
* heavy refactoring of the schema building process
* rename rproperty_keys to iter_rdef, new has_rdef method
* use dbhelper to generate index sql
* don't use ordered_relation when it's not necessary
* removed deprecated code
2007-10-29 -- 0.14.0
* schema building refactoring to read schema from a bunch of directories
* drop .perms file support
......
"""model object and utilities to define generic Entities/Relations schemas
:organization: Logilab
:copyright: 2004-2007 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
# set _ builtin to unicode by default, should be overriden if necessary
import __builtin__
__builtin__._ = unicode
from logilab.common.compat import set
BASE_TYPES = set(('String', 'Int', 'Float', 'Boolean', 'Date',
'Time', 'Datetime', 'Interval', 'Password', 'Bytes'))
from logilab.common import nullobject
MARKER = nullobject()
class FileReader(object):
"""abstract class for file readers"""
def __init__(self, loader, defaulthandler=None, readdeprecated=False):
self.loader = loader
self.default_hdlr = defaulthandler
self.read_deprecated = readdeprecated
self._current_file = None
self._current_line = None
self._current_lineno = None
def __call__(self, filepath):
self._current_file = filepath
self.read_file(filepath)
def error(self, msg=None):
"""raise a contextual exception"""
raise BadSchemaDefinition(self._current_line, self._current_file, msg)
def read_file(self, filepath):
"""default implementation, calling .read_line method for each
non blank lines, and ignoring lines starting by '#' which are
considered as comment lines
"""
for i, line in enumerate(file(filepath)):
line = line.strip()
if not line or line.startswith('#'):
continue
self._current_line = line
self._current_lineno = i
if line.startswith('//'):
if self.read_deprecated:
self.read_line(line[2:])
else:
self.read_line(line)
def read_line(self, line):
"""need overriding !"""
raise NotImplementedError()
from yams._exceptions import *
from yams.schema import Schema, EntitySchema, RelationSchema
from yams.reader import SchemaLoader
# set _ builtin to unicode by default, should be overriden if necessary
import __builtin__
__builtin__._ = unicode
"""Exceptions shared by different ER-Schema modules.
:organization: Logilab
:copyright: 2004-2007 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
......
This diff is collapsed.
......@@ -6,26 +6,25 @@ relation definitions files or a direct python definition file)
:copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
__docformat__ = "restructuredtext en"
__metaclass__ = type
import sys
from os.path import exists, join, splitext
from os import listdir
from logilab.common import attrdict
from logilab.common.fileutils import lines
from logilab.common.textutils import get_csv
from yams import UnknownType, BadSchemaDefinition
from yams import BASE_TYPES, UnknownType, BadSchemaDefinition, FileReader
from yams import constraints, schema as schemamod
from yams import builder
from yams import buildobjs
# .rel and .py formats file readers ###########################################
class RelationFileReader(builder.FileReader):
class RelationFileReader(FileReader):
"""read simple relation definitions files"""
rdefcls = builder.RelationDefinition
rdefcls = buildobjs.RelationDefinition
def read_line(self, line):
"""read a relation definition:
......@@ -52,7 +51,6 @@ class RelationFileReader(builder.FileReader):
rdef.symetric = True
relation_def.remove('symetric')
if 'inline' in relation_def:
#print 'XXX inline is deprecated'
rdef.cardinality = '?*'
rdef.inlined = True
relation_def.remove('inline')
......@@ -88,10 +86,10 @@ def _builder_context():
"""builds the context in which the schema files
will be executed
"""
return dict([(attr, getattr(builder, attr))
for attr in builder.__all__])
return dict([(attr, getattr(buildobjs, attr))
for attr in buildobjs.__all__])
class PyFileReader(builder.FileReader):
class PyFileReader(FileReader):
"""read schema definition objects from a python file"""
context = {'_' : unicode}
context.update(_builder_context())
......@@ -110,10 +108,11 @@ class PyFileReader(builder.FileReader):
if name.startswith('_'):
continue
try:
if issubclass(obj, builder.Definition):
self.loader.add_definition(self, obj())
isdef = issubclass(obj, buildobjs.Definition)
except TypeError:
continue
if isdef:
self.loader.add_definition(self, obj())
def import_schema_file(self, schemamod):
filepath = self.loader.include_schema_files(schemamod)[0]
......@@ -123,10 +122,13 @@ class PyFileReader(builder.FileReader):
return self.exec_file(filepath)
def import_erschema(self, ertype, schemamod=None, instantiate=True):
for erdef in self.loader._defobjects:
try:
erdef = self.loader.defined[ertype]
if erdef.name == ertype:
assert instantiate, 'can\'t get class of an already registered type'
return erdef
except KeyError:
pass
erdefcls = getattr(self.import_schema_file(schemamod or ertype), ertype)
if instantiate:
erdef = erdefcls()
......@@ -135,7 +137,6 @@ class PyFileReader(builder.FileReader):
return erdefcls
def exec_file(self, filepath):
#partname = self._partname(filepath)
flocals = self.context.copy()
flocals['import_schema'] = self.import_schema_file # XXX deprecate local name
flocals['import_erschema'] = self.import_erschema
......@@ -146,18 +147,10 @@ class PyFileReader(builder.FileReader):
del flocals['import_schema']
self._loaded[filepath] = attrdict(flocals)
return self._loaded[filepath]
class attrdict(dict):
"""a dictionary whose keys are also accessible as attributes"""
def __getattr__(self, attr):
try:
return self[attr]
except KeyError:
raise AttributeError(attr)
# the main schema loader ######################################################
from yams.sqlschema import EsqlFileReader
from yams.sqlreader import EsqlFileReader
class SchemaLoader(object):
"""the schema loader is responsible to build a schema object from a
......@@ -177,18 +170,9 @@ class SchemaLoader(object):
def load(self, directories, name=None, default_handler=None):
"""return a schema from the schema definition readen from <directory>
"""
self.defined = set()
self.defined = {}
self._instantiate_handlers(default_handler)
self._defobjects = []
#if self.lib_directory is not None:
# sys.path.insert(0, self.lib_directory)
#sys.path.insert(0, directory)
#try:
self._load_definition_files(directories)
#finally:
# sys.path.pop(0)
# if self.lib_directory is not None:
# sys.path.pop(0)
return self._build_schema(name)
def _instantiate_handlers(self, default_handler=None):
......@@ -205,19 +189,18 @@ class SchemaLoader(object):
def _build_schema(self, name):
"""build actual schema from definition objects, and return it"""
schema = self.schemacls(name or 'NoName')
for etype in BASE_TYPES:
edef = buildobjs.EntityType(name=etype, meta=True)
schema.add_entity_type(edef).set_default_groups()
# register relation types and non final entity types
for definition in self._defobjects:
if isinstance(definition, builder.RelationType):
definition.rschema = schema.add_relation_type(definition)
elif isinstance(definition, builder.EntityType):
definition.eschema = schema.add_entity_type(definition)
for definition in self.defined.itervalues():
if isinstance(definition, buildobjs.RelationType):
schema.add_relation_type(definition)
elif isinstance(definition, buildobjs.EntityType):
schema.add_entity_type(definition)
# register relation definitions
for definition in self._defobjects:
if isinstance(definition, builder.EntityType):
definition.register_relations(schema)
for definition in self._defobjects:
if not isinstance(definition, builder.EntityType):
definition.register_relations(schema)
for definition in self.defined.itervalues():
definition.expand_relation_definitions(self.defined, schema)
# set permissions on entities and relations
for erschema in schema.entities() + schema.relations():
erschema.set_default_groups()
......@@ -280,9 +263,13 @@ class SchemaLoader(object):
pass
def add_definition(self, hdlr, defobject):
"""file handler callback to add a definition object"""
if not isinstance(defobject, builder.Definition):
"""file handler callback to add a definition object
wildcard capability force to load schema in two steps : first register
all definition objects (here), then create actual schema objects (done in
`_build_schema`)
"""
if not isinstance(defobject, buildobjs.Definition):
hdlr.error('invalid definition object')
self.defined.add(defobject.name)
self._defobjects.append(defobject)
defobject.expand_type_definitions(self.defined)
......@@ -15,11 +15,10 @@ from logilab.common.compat import sorted
from logilab.common.interface import implements
from logilab.common.deprecation import deprecated_function
from yams import ValidationError, BadSchemaDefinition
from yams import BASE_TYPES, MARKER, ValidationError, BadSchemaDefinition
from yams.interfaces import ISchema, IRelationSchema, IEntitySchema, \
IVocabularyConstraint
from yams.constraints import BASE_CHECKERS, UniqueConstraint
from yams.builder import BASE_TYPES, EntityType
KEYWORD_MAP = {'NOW' : now,
'TODAY': today,
......@@ -70,6 +69,7 @@ def format_properties(props):
res.append('%s=%s' % (prop, value))
return ','.join(res)
class ERSchema(object):
"""common base class to entity and relation schema
"""
......@@ -80,12 +80,8 @@ class ERSchema(object):
assert erdef
self.schema = schema
self.type = erdef.name
self.meta = erdef.meta
if erdef.__doc__:
descr = ' '.join(erdef.__doc__.split())
else:
descr = ''
self.description = descr
self.meta = erdef.meta or False
self.description = erdef.description or ''
# mapping from action to groups
try:
self._groups = erdef.permissions.copy()
......@@ -394,6 +390,8 @@ class EntitySchema(ERSchema):
default = self.rproperty(rtype, 'default')
if callable(default):
default = default()
if default is MARKER:
default = None
if default is not None:
attrtype = self.destination(rtype)
if attrtype == 'Boolean':
......@@ -492,6 +490,7 @@ class EntitySchema(ERSchema):
subject_relation_schema = subject_relation
object_relation_schema = object_relation
class RelationSchema(ERSchema):
"""A relation is a named ordered link between two entities.
A relation schema defines the possible types of both extremities.
......@@ -521,8 +520,8 @@ class RelationSchema(ERSchema):
def __init__(self, schema=None, rdef=None, **kwargs):
if rdef is not None:
# if this relation is symetric/inlined
self.symetric = rdef.symetric
self.inlined = rdef.inlined
self.symetric = rdef.symetric or False
self.inlined = rdef.inlined or False
# if this relation is an attribute relation
self.final = False
# mapping to subject/object with schema as key
......@@ -638,19 +637,20 @@ class RelationSchema(ERSchema):
# relation definitions properties handling ################################
def rproperty_defs(self, desttype):
"""return a list tuple (property name, default value)
"""return a dictionary mapping property name to its definition
for each allowable properties when the relation has `desttype` as
target entity's type
"""
basekeys = self._RPROPERTIES.items()
propdefs = self._RPROPERTIES.copy()
if not self.is_final():
return basekeys + self._NONFINAL_RPROPERTIES.items()
basekeys += self._FINAL_RPROPERTIES.items()
if desttype == 'String':
return basekeys + self._STRING_RPROPERTIES.items()
if desttype == 'Bytes':
return basekeys + self._BYTES_RPROPERTIES.items()
return basekeys
propdefs.update(self._NONFINAL_RPROPERTIES)
else:
propdefs.update(self._FINAL_RPROPERTIES)
if desttype == 'String':
propdefs.update(self._STRING_RPROPERTIES)
elif desttype == 'Bytes':
propdefs.update(self._BYTES_RPROPERTIES)
return propdefs
def iter_rdefs(self):
"""return an iterator on (subject, object) of this relation"""
......@@ -673,7 +673,7 @@ class RelationSchema(ERSchema):
def set_rproperty(self, subject, object, pname, value):
"""set value for a subject relation specific property"""
#assert pname in self._rproperties
assert pname in self.rproperty_defs(object)
self._rproperties[(subject, object)][pname] = value
def init_rproperties(self, subject, object, rdef):
......@@ -682,8 +682,14 @@ class RelationSchema(ERSchema):
msg = '%s already defined for %s' % (key, self)
raise BadSchemaDefinition(msg)
self._rproperties[key] = {}
for prop, default in self.rproperty_defs(key[1]):
self._rproperties[key][prop] = getattr(rdef, prop, default)
for prop, default in self.rproperty_defs(key[1]).iteritems():
rdefval = getattr(rdef, prop, MARKER)
if rdefval is MARKER:
if prop == 'cardinality':
default = (object in BASE_TYPES) and '?1' or '**'
else:
default = rdefval
self._rproperties[key][prop] = default
# IRelationSchema interface ###############################################
......@@ -771,9 +777,6 @@ class Schema(object):
self.name = name
self._entities = {}
self._relations = {}
for etype in BASE_TYPES:
edef = EntityType(name=etype, meta=True)
self.add_entity_type(edef).set_default_groups()
def __setstate__(self, state):
self.__dict__.update(state)
......
......@@ -79,13 +79,6 @@ def eschema2sql(dbhelper, eschema, skip_relations=()):
else: # inline relation
# XXX integer is ginco specific
sqltype = 'integer'
# XXX disabled since we may want to overrides this constraint
#for oschema in eschema.destination_types(rschema.type, schema=True):
# card = rschema.rproperty(eschema.type, oschema.type, 'cardinality')
# if card[0] != '1':
# break
#else:
# sqltype += ' NOT NULL'
if i == len(attrs) - 1:
w(' %s %s' % (rschema.type, sqltype))
else:
......@@ -148,9 +141,6 @@ CREATE TABLE %(table)s (
CREATE INDEX %(table)s_from_idx ON %(table)s(eid_from);
CREATE INDEX %(table)s_to_idx ON %(table)s(eid_to);"""
# CONSTRAINT %(table)s_fkey1 FOREIGN KEY (eid_from) REFERENCES entities (eid) ON DELETE CASCADE,
# CONSTRAINT %(table)s_fkey2 FOREIGN KEY (eid_to) REFERENCES entities (eid) ON DELETE CASCADE
def rschema2sql(rschema):
return _SQL_SCHEMA % {'table': '%s_relation' % rschema.type}
......
"""Entity Schema reader, read EntitySchema from Pseudo SQL
:organization: Logilab
:copyright: 2003-2007 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:copyright: 2003-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
__docformat__ = "restructuredtext en"
import re
from os.path import split, splitext
from yams import MARKER, FileReader
from yams.constraints import StaticVocabularyConstraint, SizeConstraint, \
MultipleStaticVocabularyConstraint
from yams.builder import FileReader, EntityType, SubjectRelation
from yams.buildobjs import EntityType, SubjectRelation
FTI = 1 # Full text indexing
......@@ -103,7 +106,7 @@ class EsqlFileReader(FileReader):
"""Parses the default value and not null constraint retrieved from an
attribute definition, modifying the relation definition as necessary
"""
default = None
default = MARKER
# First, make a list for each constraint part
for cst in [cst.strip() for cst in suite.split(',') if cst.strip()]:
default_m = DEFAULT_VALUE_CST_RE.match(cst)
......@@ -116,9 +119,9 @@ class EsqlFileReader(FileReader):
rdef.cardinality = '11'
else:
self.error(cst)
if default is None:
default = getattr(self.default_hdlr, 'default_%s' % rdef.name, None)
if default is not None:
if default is MARKER:
default = getattr(self.default_hdlr, 'default_%s' % rdef.name, MARKER)
if default is not MARKER:
rdef.default = default
......
# conflicting RelationType properties
class Anentity(EntityType):
rel = SubjectRelation('Anentity', inlined=True)
class Anotherentity(EntityType):
rel = SubjectRelation('Anentity', inlined=False)
# conflicting RelationType properties
class Anentity(EntityType):
rel = SubjectRelation('Anentity', inlined=True)
class rel(RelationType):
inlined = False
# conflicting RelationType properties
class Anentity(EntityType):
rel = SubjectRelation('Anentity', inlined=True)
class rel(RelationType):
inlined = False
class otherrel(RelationType):
name = rel
inlined = False
class Anentity(EntityType):
rel = SubjectRelation('Anentity', inlined=True)
class Anotherentity(EntityType):
rel = SubjectRelation('Anentity')
class rel(RelationType):
composite = 'subject'
cardinality = '1*'
symetric = True
class __rel(RelationType):
name = 'rel'
composite = 'subject'
# -*- coding: iso-8859-1 -*-
"""unit tests for module yams.schema classes
Copyright Logilab 2004-2006, all rights reserved.
"""
"""unit tests for module yams.schema classes"""
from logilab.common.testlib import TestCase, unittest_main
from tempfile import mktemp
from yams.builder import EntityType, RelationType, RelationDefinition
from yams import BASE_TYPES
from yams.buildobjs import EntityType, RelationType, RelationDefinition
from yams.schema import *
from yams.constraints import *
......@@ -21,6 +19,9 @@ class BaseSchemaTC(TestCase):
global schema, enote, eaffaire, eperson, esociete, estring, eint
global rconcerne, rnom
schema = Schema('Test Schema')
for etype in BASE_TYPES:
edef = EntityType(name=etype, meta=True)
schema.add_entity_type(edef).set_default_groups()
enote = schema.add_entity_type(EntityType('Note'))
eaffaire = schema.add_entity_type(EntityType('Affaire'))
eperson = schema.add_entity_type(EntityType('Person'))
......@@ -142,10 +143,6 @@ class EntitySchemaTC(BaseSchemaTC):
def test_base(self):
self.assert_(repr(eperson))
# def test_is_uid(self):
# eperson.set_uid('nom')
# self.assertEquals(eperson.is_uid('nom'), True)
def test_cmp(self):
self.failUnless(eperson == 'Person')
......@@ -239,14 +236,6 @@ class EntitySchemaTC(BaseSchemaTC):
self.assertEquals(eaffaire.object_relation('concerne').type,
'concerne')
# def test_destination_types(self):
# """check subject relations a returned in the same order as in the
# schema definition"""
# expected = ['Societe']
# self.assertEquals(eperson.destination_types('travaille'), expected)
# expected = ['String']
# self.assertEquals(eperson.destination_types('nom'), expected)
def test_destination_type(self):
"""check subject relations a returned in the same order as in the
schema definition"""
......@@ -449,8 +438,7 @@ class SymetricTC(TestCase):
def test_wildcard_association_types(self):
rdef = RelationDefinition('*', 'see_also', '*')
rdef.register_relations(schema)
rdef.expand_relation_definitions({'see_also':rdef}, schema)
rsee_also = schema.rschema('see_also')
subj_types = rsee_also.associations()
subj_types.sort()
......
......@@ -181,17 +181,6 @@ class SchemaLoaderTC(TestCase):
self.assertListEquals(sorted(rschema.subjects()), ['Company', 'Division', 'Eetype', 'State'])
self.assertListEquals(sorted(rschema.objects()), ['String'])
def test_nonregr_using_tuple_as_relation_target(self):
rschema = schema.rschema('works_for')
self.assertEquals(rschema.symetric, False)
self.assertEquals(rschema.description, '')
self.assertEquals(rschema.meta, False)
self.assertEquals(rschema.is_final(), False)
self.assertListEquals(sorted(rschema.subjects()), ['Employee'])
self.assertListEquals(sorted(rschema.objects()), ['Company', 'Division'])
def test_cardinality(self):
rschema = schema.rschema('evaluee')
self.assertEquals(rschema.rproperty('Person', 'Note', 'cardinality'), '**')
......@@ -221,7 +210,7 @@ class SchemaLoaderTC(TestCase):
eschema = schema.eschema('Eetype')
self.assertEquals(len(eschema.constraints('name')), 2)
def test_physical_mode(self):
def test_inlined(self):
rschema = schema.rschema('evaluee')
self.assertEquals(rschema.inlined, False)
rschema = schema.rschema('state_of')
......@@ -231,8 +220,6 @@ class SchemaLoaderTC(TestCase):
rschema = schema.rschema('initial_state')
self.assertEquals(rschema.inlined, True)
def test_relation_permissions(self):
rschema = schema.rschema('state_of')
self.assertEquals(rschema._groups,
......@@ -285,9 +272,18 @@ class SchemaLoaderTC(TestCase):
'delete': ('managers', 'owners',),
'update': ('managers', 'owners',)})
def test_nonregr_using_tuple_as_relation_target(self):
rschema = schema.rschema('works_for')
self.assertEquals(rschema.symetric, False)
self.assertEquals(rschema.description, '')
self.assertEquals(rschema.meta, False)
self.assertEquals(rschema.is_final(), False)
self.assertListEquals(sorted(rschema.subjects()), ['Employee'])
self.assertListEquals(sorted(rschema.objects()), ['Company', 'Division'])
from yams import builder as B
from yams import