Commit 0d879692 authored by Denis Laxalde's avatar Denis Laxalde
Browse files

Merge 3.26

......@@ -614,3 +614,14 @@ d238badfc268ad4440b3238a24690858bad3fbdd debian/3.25.3-1
b8567725c473b701fe9352e578ad6e05c523c1f2 3.25.4
b8567725c473b701fe9352e578ad6e05c523c1f2 centos/3.25.4-1
b8567725c473b701fe9352e578ad6e05c523c1f2 debian/3.25.4-1
199851fcddd4b45e3d7f40efcd1739134c33db2a 3.26.0
199851fcddd4b45e3d7f40efcd1739134c33db2a debian/3.26.0-1
199851fcddd4b45e3d7f40efcd1739134c33db2a centos/3.26.0-1
027676243aaa6895492ecd31918f12d221fd503d 3.26.1
027676243aaa6895492ecd31918f12d221fd503d debian/3.26.1-1
027676243aaa6895492ecd31918f12d221fd503d centos/3.26.1-1
9bee3134d304f6cc51ab728b4ca84d464d1b3fd8 3.26.2
9bee3134d304f6cc51ab728b4ca84d464d1b3fd8 centos/3.26.2-1
9bee3134d304f6cc51ab728b4ca84d464d1b3fd8 debian/3.26.2-1
f7067be5f69cd05f34ce99fbb534e4674b3a782d 3.26.3
f7067be5f69cd05f34ce99fbb534e4674b3a782d debian/3.26.3-1
include README
include README.pyramid.rst
include COPYING
include COPYING.LESSER
include pylintrc
......@@ -31,7 +30,7 @@ include cubicweb/devtools/fix_po_encoding
recursive-include cubicweb/misc *.py *.png *.display
include cubicweb/web/views/*.pt
recursive-include cubicweb/web/data external_resources *.js *.css *.py *.png *.gif *.ico *.ttf *.svg *.woff *.eot
recursive-include cubicweb/web/data *.js *.css *.py *.png *.gif *.ico *.ttf *.svg *.woff *.eot
recursive-include cubicweb/web/wdoc *.rst *.png *.xml
recursive-include cubicweb/devtools/data *.js *.css *.sh
......
......@@ -14,7 +14,7 @@ This package contains:
Install
-------
More details at https://cubicweb.readthedocs.io/en/3.25/book/admin/setup
More details at https://cubicweb.readthedocs.io/en/3.26/book/admin/setup
Getting started
---------------
......@@ -26,12 +26,12 @@ Execute::
cubicweb-ctl start -D myblog
sensible-browser http://localhost:8080/
Details at https://cubicweb.readthedocs.io/en/3.25/tutorials/base/blog-in-five-minutes
Details at https://cubicweb.readthedocs.io/en/3.26/tutorials/base/blog-in-five-minutes
Documentation
-------------
Look in the doc/ subdirectory or read https://cubicweb.readthedocs.io/en/3.25/
Look in the doc/ subdirectory or read https://cubicweb.readthedocs.io/en/3.26/
CubicWeb includes the Entypo pictograms by Daniel Bruce — http://www.entypo.com
......
......@@ -8,7 +8,7 @@
%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
Name: cubicweb
Version: 3.25.4
Version: 3.26.3
Release: logilab.1%{?dist}
Summary: CubicWeb is a semantic web application framework
Source0: https://pypi.python.org/packages/source/c/cubicweb/cubicweb-%{version}.tar.gz
......
......@@ -19,11 +19,6 @@
"""cubicweb global packaging information for the cubicweb knowledge management
software
"""
import sys
from os import listdir
from os.path import join, isdir
import glob
modname = distname = "cubicweb"
......@@ -43,54 +38,8 @@ classifiers = [
'Programming Language :: JavaScript',
]
_server_migration_dir = join(modname, 'misc', 'migration')
_data_dir = join(modname, 'web', 'data')
_wdoc_dir = join(modname, 'web', 'wdoc')
_wdocimages_dir = join(_wdoc_dir, 'images')
_views_dir = join(modname, 'web', 'views')
_i18n_dir = join(modname, 'i18n')
_pyversion = '.'.join(str(num) for num in sys.version_info[0:2])
if '--home' in sys.argv:
# --home install
pydir = 'python' + _pyversion
else:
pydir = join('python' + _pyversion, 'site-packages')
# data files that shall be copied into the main package directory
package_data = {
'cubicweb.web.views': ['*.pt'],
'cubicweb.pyramid': ['development.ini.tmpl'],
}
try:
# data files that shall be copied outside the main package directory
data_files = [
# server data
[join('share', 'cubicweb', 'migration'),
[join(_server_migration_dir, filename)
for filename in listdir(_server_migration_dir)]],
# web data
[join('share', 'cubicweb', 'cubes', 'shared', 'data'),
[join(_data_dir, fname) for fname in listdir(_data_dir)
if not isdir(join(_data_dir, fname))]],
[join('share', 'cubicweb', 'cubes', 'shared', 'data', 'images'),
[join(_data_dir, 'images', fname) for fname in listdir(join(_data_dir, 'images'))]],
[join('share', 'cubicweb', 'cubes', 'shared', 'data', 'jquery-treeview'),
[join(_data_dir, 'jquery-treeview', fname) for fname in listdir(join(_data_dir, 'jquery-treeview'))
if not isdir(join(_data_dir, 'jquery-treeview', fname))]],
[join('share', 'cubicweb', 'cubes', 'shared', 'data', 'jquery-treeview', 'images'),
[join(_data_dir, 'jquery-treeview', 'images', fname)
for fname in listdir(join(_data_dir, 'jquery-treeview', 'images'))]],
[join('share', 'cubicweb', 'cubes', 'shared', 'wdoc'),
[join(_wdoc_dir, fname) for fname in listdir(_wdoc_dir)
if not isdir(join(_wdoc_dir, fname))]],
[join('share', 'cubicweb', 'cubes', 'shared', 'wdoc', 'images'),
[join(_wdocimages_dir, fname) for fname in listdir(_wdocimages_dir)]],
[join('share', 'cubicweb', 'cubes', 'shared', 'i18n'),
glob.glob(join(_i18n_dir, '*.po'))],
# skeleton
]
except OSError:
# we are in an installed directory, don't care about this
pass
......@@ -240,36 +240,6 @@ def guess_configuration(directory):
return modes[0]
def _find_prefix(start_path=None):
"""Return the prefix path of CubicWeb installation.
Walk parent directories of `start_path` looking for one containing a
'share/cubicweb' directory. The first matching directory is assumed as the
prefix installation of CubicWeb.
If run from within a virtualenv, the virtualenv root is used as
`start_path`. Otherwise, `start_path` defaults to cubicweb package
directory path.
"""
if start_path is None:
try:
prefix = os.environ['VIRTUAL_ENV']
except KeyError:
prefix = CW_SOFTWARE_ROOT
else:
prefix = start_path
if not isdir(prefix):
prefix = dirname(prefix)
old_prefix = None
while (not isdir(join(prefix, 'share', 'cubicweb'))
or prefix.endswith('.egg')):
if prefix == old_prefix:
return sys.prefix
old_prefix = prefix
prefix = dirname(prefix)
return prefix
def _cube_pkgname(cube):
if not cube.startswith('cubicweb_'):
return 'cubicweb_' + cube
......@@ -391,18 +361,8 @@ CFGTYPE2ETYPE_MAP = {
'float' : 'Float',
}
_forced_mode = os.environ.get('CW_MODE')
assert _forced_mode in (None, 'system', 'user')
# CWDEV tells whether directories such as i18n/, web/data/, etc. (ie containing
# some other resources than python libraries) are located with the python code
# or as a 'shared' cube
CWDEV = exists(join(CW_SOFTWARE_ROOT, 'i18n'))
try:
_INSTALL_PREFIX = os.environ['CW_INSTALL_PREFIX']
except KeyError:
_INSTALL_PREFIX = _find_prefix()
_INSTALL_PREFIX = os.environ.get('CW_INSTALL_PREFIX', sys.prefix)
_USR_INSTALL = _INSTALL_PREFIX == '/usr'
class CubicWebNoAppConfiguration(ConfigurationMixIn):
......@@ -420,15 +380,13 @@ class CubicWebNoAppConfiguration(ConfigurationMixIn):
quick_start = False
if 'VIRTUAL_ENV' in os.environ:
mode = _forced_mode or 'user'
_CUBES_DIR = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'cubes')
elif CWDEV and _forced_mode != 'system':
mode = 'user'
_CUBES_DIR = join(CW_SOFTWARE_ROOT, '../../cubes')
mode = os.environ.get('CW_MODE', 'user')
else:
mode = _forced_mode or 'system'
_CUBES_DIR = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'cubes')
mode = os.environ.get('CW_MODE', 'system')
assert mode in ('system', 'user'), '"CW_MODE" should be either "user" or "system"'
_CUBES_DIR = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'cubes')
assert _CUBES_DIR # XXX only meaningful if CW_CUBES_DIR is not set
CUBES_DIR = realpath(abspath(os.environ.get('CW_CUBES_DIR', _CUBES_DIR)))
CUBES_PATH = os.environ.get('CW_CUBES_PATH', '').split(os.pathsep)
......@@ -492,21 +450,10 @@ this option is set to yes",
def persistent_options_configuration():
return Configuration(options=PERSISTENT_OPTIONS)
@classmethod
def shared_dir(cls):
"""return the shared data directory (i.e. directory where standard
library views and data may be found)
"""
if CWDEV:
return join(CW_SOFTWARE_ROOT, 'web')
return cls.cube_dir('shared')
@classmethod
def i18n_lib_dir(cls):
"""return instance's i18n directory"""
if CWDEV:
return join(CW_SOFTWARE_ROOT, 'i18n')
return join(cls.shared_dir(), 'i18n')
return join(dirname(__file__), 'i18n')
@classmethod
def cw_languages(cls):
......@@ -1031,11 +978,8 @@ the repository',
@classmethod
def migration_scripts_dir(cls):
"""cubicweb migration scripts directory"""
if CWDEV:
return join(CW_SOFTWARE_ROOT, 'misc', 'migration')
mdir = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'migration')
if not exists(mdir):
raise ConfigurationError('migration path %s doesn\'t exist' % mdir)
mdir = join(dirname(__file__), 'misc', 'migration')
assert exists(mdir), 'migration path %s does not exist' % mdir
return mdir
@classmethod
......
......@@ -44,7 +44,7 @@ from logilab.common.configuration import merge_options
from logilab.common.decorators import clear_cache
from cubicweb import ConfigurationError, ExecutionError, BadCommandUsage
from cubicweb.cwconfig import CubicWebConfiguration as cwcfg, CWDEV, CONFIGURATIONS
from cubicweb.cwconfig import CubicWebConfiguration as cwcfg, CONFIGURATIONS
from cubicweb.toolsutils import Command, rm, create_dir, underline_title
from cubicweb.__pkginfo__ import version
......@@ -760,7 +760,7 @@ given, appropriate sources for migration will be automatically selected \
if cubicwebversion > applcubicwebversion:
toupgrade.append(('cubicweb', applcubicwebversion, cubicwebversion))
# only stop once we're sure we have something to do
if instance_running and not (CWDEV or self.config.nostartstop):
if instance_running and not self.config.nostartstop:
StopInstanceCommand(self.logger).stop_instance(appid)
# run cubicweb/componants migration scripts
if self.config.fs_only or toupgrade:
......@@ -783,7 +783,7 @@ given, appropriate sources for migration will be automatically selected \
if helper:
helper.postupgrade(repo)
print('-> instance migrated.')
if instance_running and not (CWDEV or self.config.nostartstop):
if instance_running and not self.config.nostartstop:
# restart instance through fork to get a proper environment, avoid
# uicfg pb (and probably gettext catalogs, to check...)
forkcmd = '%s start %s' % (sys.argv[0], appid)
......
......@@ -91,7 +91,10 @@ from contextlib import contextmanager
from pyramid.compat import pickle
from pyramid.session import SignedCookieSessionFactory
from cubicweb import Binary
from cubicweb import (
Binary,
UnknownEid,
)
log = logging.getLogger(__name__)
......@@ -228,8 +231,16 @@ def CWSessionFactory(
'CWSession', cwsessiondata=data)
sessioneid = session.eid
else:
session = cnx.entity_from_eid(sessioneid)
session.cw_set(cwsessiondata=data)
try:
session = cnx.entity_from_eid(sessioneid)
except UnknownEid:
# Might occur if CWSession entity got dropped (e.g.
# the whole db got recreated) while user's cookie is
# still valid. We recreate the CWSession in this case.
sessioneid = cnx.create_entity(
'CWSession', cwsessiondata=data).eid
else:
session.cw_set(cwsessiondata=data)
cnx.commit()
# Only if needed actually set the cookie
......
......@@ -1261,6 +1261,8 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
def fti_unindex_entities(self, cnx, entities):
"""remove text content for entities from the full text index
"""
if not cnx.repo.system_source.do_fti:
return
cursor = cnx.cnxset.cu
cursor_unindex_object = self.dbhelper.cursor_unindex_object
try:
......@@ -1272,6 +1274,8 @@ class NativeSQLSource(SQLAdapterMixIn, AbstractSource):
def fti_index_entities(self, cnx, entities):
"""add text content of created/modified entities to the full text index
"""
if not cnx.repo.system_source.do_fti:
return
cursor_index_object = self.dbhelper.cursor_index_object
cursor = cnx.cnxset.cu
try:
......@@ -1296,6 +1300,8 @@ class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation):
def precommit_event(self):
cnx = self.cnx
source = cnx.repo.system_source
if not source.do_fti:
return
pendingeids = cnx.transaction_data.get('pendingeids', ())
done = cnx.transaction_data.setdefault('indexedeids', set())
to_reindex = set()
......
......@@ -19,6 +19,7 @@
from __future__ import print_function
import os
import sys
import re
import subprocess
......@@ -48,13 +49,16 @@ lgc.USE_MX_DATETIME = False
SQL_PREFIX = 'cw_'
def _run_command(cmd):
def _run_command(cmd, extra_env=None):
env = os.environ.copy()
for key, value in (extra_env or {}).items():
env.setdefault(key, value)
if isinstance(cmd, string_types):
print(cmd)
return subprocess.call(cmd, shell=True)
return subprocess.call(cmd, shell=True, env=env)
else:
print(' '.join(cmd))
return subprocess.call(cmd)
return subprocess.call(cmd, env=env)
def sqlexec(sqlstmts, cursor_or_execute, withpb=True,
......@@ -342,18 +346,25 @@ class SQLAdapterMixIn(object):
"""open and return a connection to the database"""
return self.dbhelper.get_connection()
def _backup_restore_env(self):
if (self.config['db-driver'] == 'postgres'
and self.config['db-password'] is not None):
return {'PGPASSWORD': self.config['db-password']}
def backup_to_file(self, backupfile, confirm):
extra_env = self._backup_restore_env()
for cmd in self.dbhelper.backup_commands(backupfile,
keepownership=False):
if _run_command(cmd):
if _run_command(cmd, extra_env=extra_env):
if not confirm(' [Failed] Continue anyway?', default='n'):
raise Exception('Failed command: %s' % cmd)
def restore_from_file(self, backupfile, confirm, drop=True):
extra_env = self._backup_restore_env()
for cmd in self.dbhelper.restore_commands(backupfile,
keepownership=False,
drop=drop):
if _run_command(cmd):
if _run_command(cmd, extra_env=extra_env):
if not confirm(' [Failed] Continue anyway?', default='n'):
raise Exception('Failed command: %s' % cmd)
......
......@@ -26,7 +26,7 @@ import logging
from threading import Thread
from getpass import getpass
from six import PY2
from six import PY2, text_type
from six.moves import input
from passlib.utils import handlers as uh, to_hash_str
......@@ -106,7 +106,7 @@ def manager_userpasswd(user=None, msg=DEFAULT_MSG, confirm=False,
while not user:
user = input('login: ')
if PY2:
user = unicode(user, sys.stdin.encoding)
user = text_type(user, sys.stdin.encoding)
passwd = getpass('%s: ' % passwdmsg)
if confirm:
while True:
......
......@@ -47,7 +47,7 @@ There are 3 kinds of statds_ message::
There is also a decorator (``statsd_timeit``) that may be used to
measure and send to the statsd_ server the time passed in a function
or a method and the number of calls. It will send a message like::
<bucket>.<funcname>:<ms>|ms\n<bucket>.<funcname>:1|c\n
......@@ -56,7 +56,6 @@ or a method and the number of calls. It will send a message like::
"""
import time
import socket
......@@ -112,10 +111,11 @@ class statsd_timeit(object):
@property
def __doc__(self):
return self.callable.__doc__
@property
def __name__(self):
return self.callable.__name__
def __call__(self, *args, **kw):
if _address is None:
return self.callable(*args, **kw)
......@@ -123,13 +123,14 @@ class statsd_timeit(object):
try:
return self.callable(*args, **kw)
finally:
dt = 1000*(time.time()-t0)
msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format(_bucket, self.__name__, dt)
dt = 1000 * (time.time() - t0)
msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format(
_bucket, self.__name__, dt)
_socket.sendto(msg, _address)
def __get__(self, obj, objtype):
"""Support instance methods."""
if obj is None: # class method or some already wrapped method
if obj is None: # class method or some already wrapped method
return self
import functools
return functools.partial(self.__call__, obj)
......@@ -35,17 +35,7 @@ from logilab.common.modutils import cleanup_sys_modules
from cubicweb.devtools import ApptestConfiguration
from cubicweb.devtools.testlib import BaseTestCase, TemporaryDirectory
from cubicweb.cwconfig import (
CubicWebConfiguration, _find_prefix, _expand_modname)
def unabsolutize(path):
parts = path.split(os.sep)
for i, part in reversed(tuple(enumerate(parts))):
if part.startswith('cubicweb_'):
return os.sep.join([part[len('cubicweb_'):]] + parts[i + 1:])
if part.startswith('cubicweb') or part == 'legacy_cubes':
return os.sep.join(parts[i + 1:])
raise Exception('duh? %s' % path)
CubicWebConfiguration, _expand_modname)
def templibdir(func):
......@@ -125,6 +115,12 @@ class CubicWebConfigurationTC(BaseTestCase):
ApptestConfiguration.CUBES_PATH = []
cleanup_sys_modules([self.datapath('libpython')])
def test_migration_scripts_dir(self):
mscripts = os.listdir(self.config.migration_scripts_dir())
self.assertIn('bootstrapmigration_repository.py', mscripts)
self.assertIn('postcreate.py', mscripts)
self.assertIn('3.24.0_Any.py', mscripts)
@patch('pkg_resources.iter_entry_points', side_effect=iter_entry_points)
def test_available_cubes(self, mock_iter_entry_points):
expected_cubes = [
......@@ -289,101 +285,6 @@ class CubicWebConfigurationWithLegacyCubesTC(CubicWebConfigurationTC):
self.assertNotIn('cubicweb_mycube.ccplugin', sys.modules, sorted(sys.modules))
class FindPrefixTC(unittest.TestCase):
def make_dirs(self, basedir, *args):
path = join(basedir, *args)
if not os.path.exists(path):
os.makedirs(path)
return path
def make_file(self, basedir, *args):
self.make_dirs(basedir, *args[:-1])
file_path = join(basedir, *args)
with open(file_path, 'w') as f:
f.write('""" None """')
return file_path
def test_samedir(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
self.assertEqual(_find_prefix(prefix), prefix)
def test_samedir_filepath(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
file_path = self.make_file(prefix, 'bob.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_dir_inside_prefix(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
dir_path = self.make_dirs(prefix, 'bob')
self.assertEqual(_find_prefix(dir_path), prefix)
def test_file_in_dir_inside_prefix(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
file_path = self.make_file(prefix, 'bob', 'toto.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_file_in_deeper_dir_inside_prefix(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
file_path = self.make_file(prefix, 'bob', 'pyves', 'alain',
'adim', 'syt', 'toto.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_multiple_candidate_prefix(self):
with TemporaryDirectory() as tempdir:
self.make_dirs(tempdir, 'share', 'cubicweb')
prefix = self.make_dirs(tempdir, 'bob')
self.make_dirs(prefix, 'share', 'cubicweb')
file_path = self.make_file(prefix, 'pyves', 'alain',
'adim', 'syt', 'toto.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_sister_candidate_prefix(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
self.make_dirs(prefix, 'bob', 'share', 'cubicweb')
file_path = self.make_file(prefix, 'bell', 'toto.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_multiple_parent_candidate_prefix(self):
with TemporaryDirectory() as tempdir:
self.make_dirs(tempdir, 'share', 'cubicweb')
prefix = self.make_dirs(tempdir, 'share', 'cubicweb', 'bob')
self.make_dirs(tempdir, 'share', 'cubicweb', 'bob', 'share',
'cubicweb')
file_path = self.make_file(tempdir, 'share', 'cubicweb', 'bob',
'pyves', 'alain', 'adim', 'syt',
'toto.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_upper_candidate_prefix(self):
with TemporaryDirectory() as prefix:
self.make_dirs(prefix, 'share', 'cubicweb')
self.make_dirs(prefix, 'bell', 'bob', 'share', 'cubicweb')
file_path = self.make_file(prefix, 'bell', 'toto.py')
self.assertEqual(_find_prefix(file_path), prefix)
def test_no_prefix(self):
with TemporaryDirectory() as prefix:
self.assertEqual(_find_prefix(prefix), sys.prefix)
def test_virtualenv(self):
venv = os.environ.get('VIRTUAL_ENV')
try:
with TemporaryDirectory() as prefix:
os.environ['VIRTUAL_ENV'] = prefix
self.make_dirs(prefix, 'share', 'cubicweb')
self.assertEqual(_find_prefix(), prefix)
finally:
if venv:
os.environ['VIRTUAL_ENV'] = venv
class ModnamesTC(unittest.TestCase):
@templibdir
......
......@@ -108,6 +108,12 @@ class PropertySheet(dict):
tmpfd, tmpfile = tempfile.mkstemp(dir=rcachedir, prefix=osp.basename(cachefile))
with os.fdopen(tmpfd, 'w') as stream:
stream.write(content)
try:
mode = os.stat(sourcefile).st_mode
os.chmod(tmpfile, mode)
except IOError:
self.warning('Cannot set access mode for %s; you may encouter '
'file permissions issues', cachefile)
try:
os.rename(tmpfile, cachefile)
except OSError as err:
......
......@@ -40,13 +40,13 @@ class PropertySheetTC(TestCase):
self.assertEqual(ps['fontcolor'], 'black')
# defined by sheet1, extended by sheet2
self.assertEqual(ps['stylesheets'], ['http://cwtest.com/cubicweb.css',
'http://cwtest.com/mycube.css'])