Commit f3d758cd authored by Sylvain's avatar Sylvain
Browse files

schema building refactoring

parent ad6bcf3df1e8
...@@ -2,8 +2,12 @@ ChangeLog for yams ...@@ -2,8 +2,12 @@ ChangeLog for yams
------------------ ------------------
-- --
* heavy refactoring of the schema building process
* rename rproperty_keys to iter_rdef, new has_rdef method
* use dbhelper to generate index sql
* don't use ordered_relation when it's not necessary
* removed deprecated code * removed deprecated code
2007-10-29 -- 0.14.0 2007-10-29 -- 0.14.0
* schema building refactoring to read schema from a bunch of directories * schema building refactoring to read schema from a bunch of directories
* drop .perms file support * drop .perms file support
......
"""model object and utilities to define generic Entities/Relations schemas """model object and utilities to define generic Entities/Relations schemas
:organization: Logilab :organization: Logilab
:copyright: 2004-2007 LOGILAB S.A. (Paris, FRANCE), all rights reserved. :copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
""" """
# set _ builtin to unicode by default, should be overriden if necessary
import __builtin__
__builtin__._ = unicode
from logilab.common.compat import set
BASE_TYPES = set(('String', 'Int', 'Float', 'Boolean', 'Date',
'Time', 'Datetime', 'Interval', 'Password', 'Bytes'))
from logilab.common import nullobject
MARKER = nullobject()
class FileReader(object):
"""abstract class for file readers"""
def __init__(self, loader, defaulthandler=None, readdeprecated=False):
self.loader = loader
self.default_hdlr = defaulthandler
self.read_deprecated = readdeprecated
self._current_file = None
self._current_line = None
self._current_lineno = None
def __call__(self, filepath):
self._current_file = filepath
self.read_file(filepath)
def error(self, msg=None):
"""raise a contextual exception"""
raise BadSchemaDefinition(self._current_line, self._current_file, msg)
def read_file(self, filepath):
"""default implementation, calling .read_line method for each
non blank lines, and ignoring lines starting by '#' which are
considered as comment lines
"""
for i, line in enumerate(file(filepath)):
line = line.strip()
if not line or line.startswith('#'):
continue
self._current_line = line
self._current_lineno = i
if line.startswith('//'):
if self.read_deprecated:
self.read_line(line[2:])
else:
self.read_line(line)
def read_line(self, line):
"""need overriding !"""
raise NotImplementedError()
from yams._exceptions import * from yams._exceptions import *
from yams.schema import Schema, EntitySchema, RelationSchema from yams.schema import Schema, EntitySchema, RelationSchema
from yams.reader import SchemaLoader from yams.reader import SchemaLoader
# set _ builtin to unicode by default, should be overriden if necessary
import __builtin__
__builtin__._ = unicode
"""Exceptions shared by different ER-Schema modules. """Exceptions shared by different ER-Schema modules.
:organization: Logilab :organization: Logilab
:copyright: 2004-2007 LOGILAB S.A. (Paris, FRANCE), all rights reserved. :copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
""" """
......
This diff is collapsed.
...@@ -6,26 +6,25 @@ relation definitions files or a direct python definition file) ...@@ -6,26 +6,25 @@ relation definitions files or a direct python definition file)
:copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved. :copyright: 2004-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
""" """
__docformat__ = "restructuredtext en" __docformat__ = "restructuredtext en"
__metaclass__ = type
import sys import sys
from os.path import exists, join, splitext from os.path import exists, join, splitext
from os import listdir from os import listdir
from logilab.common import attrdict
from logilab.common.fileutils import lines from logilab.common.fileutils import lines
from logilab.common.textutils import get_csv from logilab.common.textutils import get_csv
from yams import UnknownType, BadSchemaDefinition from yams import BASE_TYPES, UnknownType, BadSchemaDefinition, FileReader
from yams import constraints, schema as schemamod from yams import constraints, schema as schemamod
from yams import builder from yams import buildobjs
# .rel and .py formats file readers ########################################### # .rel and .py formats file readers ###########################################
class RelationFileReader(builder.FileReader): class RelationFileReader(FileReader):
"""read simple relation definitions files""" """read simple relation definitions files"""
rdefcls = builder.RelationDefinition rdefcls = buildobjs.RelationDefinition
def read_line(self, line): def read_line(self, line):
"""read a relation definition: """read a relation definition:
...@@ -52,7 +51,6 @@ class RelationFileReader(builder.FileReader): ...@@ -52,7 +51,6 @@ class RelationFileReader(builder.FileReader):
rdef.symetric = True rdef.symetric = True
relation_def.remove('symetric') relation_def.remove('symetric')
if 'inline' in relation_def: if 'inline' in relation_def:
#print 'XXX inline is deprecated'
rdef.cardinality = '?*' rdef.cardinality = '?*'
rdef.inlined = True rdef.inlined = True
relation_def.remove('inline') relation_def.remove('inline')
...@@ -88,10 +86,10 @@ def _builder_context(): ...@@ -88,10 +86,10 @@ def _builder_context():
"""builds the context in which the schema files """builds the context in which the schema files
will be executed will be executed
""" """
return dict([(attr, getattr(builder, attr)) return dict([(attr, getattr(buildobjs, attr))
for attr in builder.__all__]) for attr in buildobjs.__all__])
class PyFileReader(builder.FileReader): class PyFileReader(FileReader):
"""read schema definition objects from a python file""" """read schema definition objects from a python file"""
context = {'_' : unicode} context = {'_' : unicode}
context.update(_builder_context()) context.update(_builder_context())
...@@ -110,10 +108,11 @@ class PyFileReader(builder.FileReader): ...@@ -110,10 +108,11 @@ class PyFileReader(builder.FileReader):
if name.startswith('_'): if name.startswith('_'):
continue continue
try: try:
if issubclass(obj, builder.Definition): isdef = issubclass(obj, buildobjs.Definition)
self.loader.add_definition(self, obj())
except TypeError: except TypeError:
continue continue
if isdef:
self.loader.add_definition(self, obj())
def import_schema_file(self, schemamod): def import_schema_file(self, schemamod):
filepath = self.loader.include_schema_files(schemamod)[0] filepath = self.loader.include_schema_files(schemamod)[0]
...@@ -123,10 +122,13 @@ class PyFileReader(builder.FileReader): ...@@ -123,10 +122,13 @@ class PyFileReader(builder.FileReader):
return self.exec_file(filepath) return self.exec_file(filepath)
def import_erschema(self, ertype, schemamod=None, instantiate=True): def import_erschema(self, ertype, schemamod=None, instantiate=True):
for erdef in self.loader._defobjects: try:
erdef = self.loader.defined[ertype]
if erdef.name == ertype: if erdef.name == ertype:
assert instantiate, 'can\'t get class of an already registered type' assert instantiate, 'can\'t get class of an already registered type'
return erdef return erdef
except KeyError:
pass
erdefcls = getattr(self.import_schema_file(schemamod or ertype), ertype) erdefcls = getattr(self.import_schema_file(schemamod or ertype), ertype)
if instantiate: if instantiate:
erdef = erdefcls() erdef = erdefcls()
...@@ -135,7 +137,6 @@ class PyFileReader(builder.FileReader): ...@@ -135,7 +137,6 @@ class PyFileReader(builder.FileReader):
return erdefcls return erdefcls
def exec_file(self, filepath): def exec_file(self, filepath):
#partname = self._partname(filepath)
flocals = self.context.copy() flocals = self.context.copy()
flocals['import_schema'] = self.import_schema_file # XXX deprecate local name flocals['import_schema'] = self.import_schema_file # XXX deprecate local name
flocals['import_erschema'] = self.import_erschema flocals['import_erschema'] = self.import_erschema
...@@ -146,18 +147,10 @@ class PyFileReader(builder.FileReader): ...@@ -146,18 +147,10 @@ class PyFileReader(builder.FileReader):
del flocals['import_schema'] del flocals['import_schema']
self._loaded[filepath] = attrdict(flocals) self._loaded[filepath] = attrdict(flocals)
return self._loaded[filepath] return self._loaded[filepath]
class attrdict(dict):
"""a dictionary whose keys are also accessible as attributes"""
def __getattr__(self, attr):
try:
return self[attr]
except KeyError:
raise AttributeError(attr)
# the main schema loader ###################################################### # the main schema loader ######################################################
from yams.sqlschema import EsqlFileReader from yams.sqlreader import EsqlFileReader
class SchemaLoader(object): class SchemaLoader(object):
"""the schema loader is responsible to build a schema object from a """the schema loader is responsible to build a schema object from a
...@@ -177,18 +170,9 @@ class SchemaLoader(object): ...@@ -177,18 +170,9 @@ class SchemaLoader(object):
def load(self, directories, name=None, default_handler=None): def load(self, directories, name=None, default_handler=None):
"""return a schema from the schema definition readen from <directory> """return a schema from the schema definition readen from <directory>
""" """
self.defined = set() self.defined = {}
self._instantiate_handlers(default_handler) self._instantiate_handlers(default_handler)
self._defobjects = []
#if self.lib_directory is not None:
# sys.path.insert(0, self.lib_directory)
#sys.path.insert(0, directory)
#try:
self._load_definition_files(directories) self._load_definition_files(directories)
#finally:
# sys.path.pop(0)
# if self.lib_directory is not None:
# sys.path.pop(0)
return self._build_schema(name) return self._build_schema(name)
def _instantiate_handlers(self, default_handler=None): def _instantiate_handlers(self, default_handler=None):
...@@ -205,19 +189,18 @@ class SchemaLoader(object): ...@@ -205,19 +189,18 @@ class SchemaLoader(object):
def _build_schema(self, name): def _build_schema(self, name):
"""build actual schema from definition objects, and return it""" """build actual schema from definition objects, and return it"""
schema = self.schemacls(name or 'NoName') schema = self.schemacls(name or 'NoName')
for etype in BASE_TYPES:
edef = buildobjs.EntityType(name=etype, meta=True)
schema.add_entity_type(edef).set_default_groups()
# register relation types and non final entity types # register relation types and non final entity types
for definition in self._defobjects: for definition in self.defined.itervalues():
if isinstance(definition, builder.RelationType): if isinstance(definition, buildobjs.RelationType):
definition.rschema = schema.add_relation_type(definition) schema.add_relation_type(definition)
elif isinstance(definition, builder.EntityType): elif isinstance(definition, buildobjs.EntityType):
definition.eschema = schema.add_entity_type(definition) schema.add_entity_type(definition)
# register relation definitions # register relation definitions
for definition in self._defobjects: for definition in self.defined.itervalues():
if isinstance(definition, builder.EntityType): definition.expand_relation_definitions(self.defined, schema)
definition.register_relations(schema)
for definition in self._defobjects:
if not isinstance(definition, builder.EntityType):
definition.register_relations(schema)
# set permissions on entities and relations # set permissions on entities and relations
for erschema in schema.entities() + schema.relations(): for erschema in schema.entities() + schema.relations():
erschema.set_default_groups() erschema.set_default_groups()
...@@ -280,9 +263,13 @@ class SchemaLoader(object): ...@@ -280,9 +263,13 @@ class SchemaLoader(object):
pass pass
def add_definition(self, hdlr, defobject): def add_definition(self, hdlr, defobject):
"""file handler callback to add a definition object""" """file handler callback to add a definition object
if not isinstance(defobject, builder.Definition):
wildcard capability force to load schema in two steps : first register
all definition objects (here), then create actual schema objects (done in
`_build_schema`)
"""
if not isinstance(defobject, buildobjs.Definition):
hdlr.error('invalid definition object') hdlr.error('invalid definition object')
self.defined.add(defobject.name) defobject.expand_type_definitions(self.defined)
self._defobjects.append(defobject)
...@@ -15,11 +15,10 @@ from logilab.common.compat import sorted ...@@ -15,11 +15,10 @@ from logilab.common.compat import sorted
from logilab.common.interface import implements from logilab.common.interface import implements
from logilab.common.deprecation import deprecated_function from logilab.common.deprecation import deprecated_function
from yams import ValidationError, BadSchemaDefinition from yams import BASE_TYPES, MARKER, ValidationError, BadSchemaDefinition
from yams.interfaces import ISchema, IRelationSchema, IEntitySchema, \ from yams.interfaces import ISchema, IRelationSchema, IEntitySchema, \
IVocabularyConstraint IVocabularyConstraint
from yams.constraints import BASE_CHECKERS, UniqueConstraint from yams.constraints import BASE_CHECKERS, UniqueConstraint
from yams.builder import BASE_TYPES, EntityType
KEYWORD_MAP = {'NOW' : now, KEYWORD_MAP = {'NOW' : now,
'TODAY': today, 'TODAY': today,
...@@ -70,6 +69,7 @@ def format_properties(props): ...@@ -70,6 +69,7 @@ def format_properties(props):
res.append('%s=%s' % (prop, value)) res.append('%s=%s' % (prop, value))
return ','.join(res) return ','.join(res)
class ERSchema(object): class ERSchema(object):
"""common base class to entity and relation schema """common base class to entity and relation schema
""" """
...@@ -80,12 +80,8 @@ class ERSchema(object): ...@@ -80,12 +80,8 @@ class ERSchema(object):
assert erdef assert erdef
self.schema = schema self.schema = schema
self.type = erdef.name self.type = erdef.name
self.meta = erdef.meta self.meta = erdef.meta or False
if erdef.__doc__: self.description = erdef.description or ''
descr = ' '.join(erdef.__doc__.split())
else:
descr = ''
self.description = descr
# mapping from action to groups # mapping from action to groups
try: try:
self._groups = erdef.permissions.copy() self._groups = erdef.permissions.copy()
...@@ -394,6 +390,8 @@ class EntitySchema(ERSchema): ...@@ -394,6 +390,8 @@ class EntitySchema(ERSchema):
default = self.rproperty(rtype, 'default') default = self.rproperty(rtype, 'default')
if callable(default): if callable(default):
default = default() default = default()
if default is MARKER:
default = None
if default is not None: if default is not None:
attrtype = self.destination(rtype) attrtype = self.destination(rtype)
if attrtype == 'Boolean': if attrtype == 'Boolean':
...@@ -492,6 +490,7 @@ class EntitySchema(ERSchema): ...@@ -492,6 +490,7 @@ class EntitySchema(ERSchema):
subject_relation_schema = subject_relation subject_relation_schema = subject_relation
object_relation_schema = object_relation object_relation_schema = object_relation
class RelationSchema(ERSchema): class RelationSchema(ERSchema):
"""A relation is a named ordered link between two entities. """A relation is a named ordered link between two entities.
A relation schema defines the possible types of both extremities. A relation schema defines the possible types of both extremities.
...@@ -521,8 +520,8 @@ class RelationSchema(ERSchema): ...@@ -521,8 +520,8 @@ class RelationSchema(ERSchema):
def __init__(self, schema=None, rdef=None, **kwargs): def __init__(self, schema=None, rdef=None, **kwargs):
if rdef is not None: if rdef is not None:
# if this relation is symetric/inlined # if this relation is symetric/inlined
self.symetric = rdef.symetric self.symetric = rdef.symetric or False
self.inlined = rdef.inlined self.inlined = rdef.inlined or False
# if this relation is an attribute relation # if this relation is an attribute relation
self.final = False self.final = False
# mapping to subject/object with schema as key # mapping to subject/object with schema as key
...@@ -638,19 +637,20 @@ class RelationSchema(ERSchema): ...@@ -638,19 +637,20 @@ class RelationSchema(ERSchema):
# relation definitions properties handling ################################ # relation definitions properties handling ################################
def rproperty_defs(self, desttype): def rproperty_defs(self, desttype):
"""return a list tuple (property name, default value) """return a dictionary mapping property name to its definition
for each allowable properties when the relation has `desttype` as for each allowable properties when the relation has `desttype` as
target entity's type target entity's type
""" """
basekeys = self._RPROPERTIES.items() propdefs = self._RPROPERTIES.copy()
if not self.is_final(): if not self.is_final():
return basekeys + self._NONFINAL_RPROPERTIES.items() propdefs.update(self._NONFINAL_RPROPERTIES)
basekeys += self._FINAL_RPROPERTIES.items() else:
if desttype == 'String': propdefs.update(self._FINAL_RPROPERTIES)
return basekeys + self._STRING_RPROPERTIES.items() if desttype == 'String':
if desttype == 'Bytes': propdefs.update(self._STRING_RPROPERTIES)
return basekeys + self._BYTES_RPROPERTIES.items() elif desttype == 'Bytes':
return basekeys propdefs.update(self._BYTES_RPROPERTIES)
return propdefs
def iter_rdefs(self): def iter_rdefs(self):
"""return an iterator on (subject, object) of this relation""" """return an iterator on (subject, object) of this relation"""
...@@ -673,7 +673,7 @@ class RelationSchema(ERSchema): ...@@ -673,7 +673,7 @@ class RelationSchema(ERSchema):
def set_rproperty(self, subject, object, pname, value): def set_rproperty(self, subject, object, pname, value):
"""set value for a subject relation specific property""" """set value for a subject relation specific property"""
#assert pname in self._rproperties assert pname in self.rproperty_defs(object)
self._rproperties[(subject, object)][pname] = value self._rproperties[(subject, object)][pname] = value
def init_rproperties(self, subject, object, rdef): def init_rproperties(self, subject, object, rdef):
...@@ -682,8 +682,14 @@ class RelationSchema(ERSchema): ...@@ -682,8 +682,14 @@ class RelationSchema(ERSchema):
msg = '%s already defined for %s' % (key, self) msg = '%s already defined for %s' % (key, self)
raise BadSchemaDefinition(msg) raise BadSchemaDefinition(msg)
self._rproperties[key] = {} self._rproperties[key] = {}
for prop, default in self.rproperty_defs(key[1]): for prop, default in self.rproperty_defs(key[1]).iteritems():
self._rproperties[key][prop] = getattr(rdef, prop, default) rdefval = getattr(rdef, prop, MARKER)
if rdefval is MARKER:
if prop == 'cardinality':
default = (object in BASE_TYPES) and '?1' or '**'
else:
default = rdefval
self._rproperties[key][prop] = default
# IRelationSchema interface ############################################### # IRelationSchema interface ###############################################
...@@ -771,9 +777,6 @@ class Schema(object): ...@@ -771,9 +777,6 @@ class Schema(object):
self.name = name self.name = name
self._entities = {} self._entities = {}
self._relations = {} self._relations = {}
for etype in BASE_TYPES:
edef = EntityType(name=etype, meta=True)
self.add_entity_type(edef).set_default_groups()
def __setstate__(self, state): def __setstate__(self, state):
self.__dict__.update(state) self.__dict__.update(state)
......
...@@ -79,13 +79,6 @@ def eschema2sql(dbhelper, eschema, skip_relations=()): ...@@ -79,13 +79,6 @@ def eschema2sql(dbhelper, eschema, skip_relations=()):
else: # inline relation else: # inline relation
# XXX integer is ginco specific # XXX integer is ginco specific
sqltype = 'integer' sqltype = 'integer'
# XXX disabled since we may want to overrides this constraint
#for oschema in eschema.destination_types(rschema.type, schema=True):
# card = rschema.rproperty(eschema.type, oschema.type, 'cardinality')
# if card[0] != '1':
# break
#else:
# sqltype += ' NOT NULL'
if i == len(attrs) - 1: if i == len(attrs) - 1:
w(' %s %s' % (rschema.type, sqltype)) w(' %s %s' % (rschema.type, sqltype))
else: else:
...@@ -148,9 +141,6 @@ CREATE TABLE %(table)s ( ...@@ -148,9 +141,6 @@ CREATE TABLE %(table)s (
CREATE INDEX %(table)s_from_idx ON %(table)s(eid_from); CREATE INDEX %(table)s_from_idx ON %(table)s(eid_from);
CREATE INDEX %(table)s_to_idx ON %(table)s(eid_to);""" CREATE INDEX %(table)s_to_idx ON %(table)s(eid_to);"""
# CONSTRAINT %(table)s_fkey1 FOREIGN KEY (eid_from) REFERENCES entities (eid) ON DELETE CASCADE,
# CONSTRAINT %(table)s_fkey2 FOREIGN KEY (eid_to) REFERENCES entities (eid) ON DELETE CASCADE
def rschema2sql(rschema): def rschema2sql(rschema):
return _SQL_SCHEMA % {'table': '%s_relation' % rschema.type} return _SQL_SCHEMA % {'table': '%s_relation' % rschema.type}
......
"""Entity Schema reader, read EntitySchema from Pseudo SQL """Entity Schema reader, read EntitySchema from Pseudo SQL
:organization: Logilab :organization: Logilab
:copyright: 2003-2007 LOGILAB S.A. (Paris, FRANCE), all rights reserved. :copyright: 2003-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
""" """
__docformat__ = "restructuredtext en"
import re import re
from os.path import split, splitext from os.path import split, splitext
from yams import MARKER, FileReader
from yams.constraints import StaticVocabularyConstraint, SizeConstraint, \ from yams.constraints import StaticVocabularyConstraint, SizeConstraint, \
MultipleStaticVocabularyConstraint MultipleStaticVocabularyConstraint
from yams.builder import FileReader, EntityType, SubjectRelation from yams.buildobjs import EntityType, SubjectRelation
FTI = 1 # Full text indexing FTI = 1 # Full text indexing
...@@ -103,7 +106,7 @@ class EsqlFileReader(FileReader): ...@@ -103,7 +106,7 @@ class EsqlFileReader(FileReader):
"""Parses the default value and not null constraint retrieved from an """Parses the default value and not null constraint retrieved from an
attribute definition, modifying the relation definition as necessary attribute definition, modifying the relation definition as necessary
""" """
default = None default = MARKER
# First, make a list for each constraint part # First, make a list for each constraint part
for cst in [cst.strip() for cst in suite.split(',') if cst.strip()]: for cst in [cst.strip() for cst in suite.split(',') if cst.strip()]:
default_m = DEFAULT_VALUE_CST_RE.match(cst) default_m = DEFAULT_VALUE_CST_RE.match(cst)
...@@ -116,9 +119,9 @@ class EsqlFileReader(FileReader): ...@@ -116,9 +119,9 @@ class EsqlFileReader(FileReader):
rdef.cardinality = '11' rdef.cardinality = '11'
else: else:
self.error(cst) self.error(cst)
if default is None: if default is MARKER:
default = getattr(self.default_hdlr, 'default_%s' % rdef.name, None) default = getattr(self.default_hdlr, 'default_%s' % rdef.name, MARKER)
if default is not None: if default is not MARKER:
rdef.default = default rdef.default = default
......