Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
cubicweb
cubes
dataprocessing
Commits
40d0a4586fb2
Commit
d879fcee
authored
Oct 19, 2021
by
François Ferry
Browse files
feat(2to3): run 2to3 to convert some py2 syntaxes
parent
efa67c047fd8
Changes
12
Hide whitespace changes
Inline
Side-by-side
cubicweb_dataprocessing/entities.py
View file @
40d0a458
...
...
@@ -46,7 +46,7 @@ class TransformationStep(AnyEntity):
def
dc_title
(
self
):
script
=
self
.
step_script
[
0
]
return
u
"[{0}] {1}"
.
format
(
self
.
index
,
script
.
dc_title
())
return
"[{0}] {1}"
.
format
(
self
.
index
,
script
.
dc_title
())
class
DataProcessAdapter
(
EntityAdapter
):
...
...
@@ -77,7 +77,7 @@ class DataProcessAdapter(EntityAdapter):
cmdline
=
[
sys
.
executable
,
scriptpath
,
inputfpath
]
if
parameters
:
params
=
json
.
loads
(
parameters
)
for
pname
,
pvalue
in
params
.
items
():
for
pname
,
pvalue
in
list
(
params
.
items
()
)
:
if
not
isinstance
(
pvalue
,
text_type
):
raise
ValueError
(
'invalid parameter value for "{0}": {1}, '
...
...
cubicweb_dataprocessing/hooks.py
View file @
40d0a458
...
...
@@ -15,7 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb-dataprocessing specific hooks and operations"""
from
__future__
import
print_function
import
os
import
os.path
...
...
@@ -143,7 +143,7 @@ class ExecuteProcessScriptsOp(hook.DataOperationMixIn, hook.Operation):
process
=
self
.
cnx
.
entity_from_eid
(
peid
)
iprocess
=
process
.
cw_adapt_to
(
"IDataProcess"
)
if
not
process
.
process_input_file
:
msg
=
u
"no input file"
msg
=
"no input file"
iprocess
.
fire_workflow_transition
(
"error"
,
comment
=
msg
)
return
inputfile
=
process
.
process_input_file
[
0
]
...
...
@@ -155,7 +155,7 @@ class ExecuteProcessScriptsOp(hook.DataOperationMixIn, hook.Operation):
script
,
inputfpath
,
outfile
,
params
)
if
returncode
:
msg
=
u
"
\n
"
.
join
(
msg
=
"
\n
"
.
join
(
[
"error in transformation #%d of input file #%d"
%
(
idx
,
inputfile
.
eid
),
...
...
@@ -165,8 +165,8 @@ class ExecuteProcessScriptsOp(hook.DataOperationMixIn, hook.Operation):
self
.
cnx
.
create_entity
(
"File"
,
data
=
Binary
(
stderr
),
data_name
=
u
"stderr"
,
data_format
=
u
"text/plain"
,
data_name
=
"stderr"
,
data_format
=
"text/plain"
,
reverse_process_stderr
=
process
,
)
iprocess
.
fire_workflow_transition
(
"error"
,
comment
=
msg
)
...
...
cubicweb_dataprocessing/views.py
View file @
40d0a458
...
...
@@ -61,7 +61,7 @@ class JSONAttributeView(EntityView):
attr
=
getattr
(
entity
,
rtype
)
if
attr
:
js
=
json
.
dumps
(
json
.
loads
(
attr
),
indent
=
2
)
self
.
w
(
u
"<pre>{0}</pre>"
.
format
(
js
))
self
.
w
(
"<pre>{0}</pre>"
.
format
(
js
))
for
etype
in
(
"ValidationScript"
,
"TransformationScript"
):
...
...
cubicweb_dataprocessing/workflows.py
View file @
40d0a458
...
...
@@ -21,7 +21,7 @@ from cubicweb import _
def
define_dataprocess_workflow
(
add_workflow
):
"""Define workflow for data process entity types"""
etypes
=
"DataTransformationProcess"
,
"DataValidationProcess"
wf
=
add_workflow
(
u
"Data processing workflow"
,
etypes
)
wf
=
add_workflow
(
"Data processing workflow"
,
etypes
)
# States
initialized
=
wf
.
add_state
(
_
(
"wfs_dataprocess_initialized"
),
initial
=
True
)
in_progress
=
wf
.
add_state
(
_
(
"wfs_dataprocess_in_progress"
))
...
...
setup.py
View file @
40d0a458
...
...
@@ -58,7 +58,7 @@ requires = {}
for
entry
in
(
"__depends__"
,):
# "__recommends__"):
requires
.
update
(
__pkginfo__
.
get
(
entry
,
{}))
install_requires
=
[
"{0} {1}"
.
format
(
d
,
v
and
v
or
""
).
strip
()
for
d
,
v
in
requires
.
items
()
"{0} {1}"
.
format
(
d
,
v
and
v
or
""
).
strip
()
for
d
,
v
in
list
(
requires
.
items
()
)
]
...
...
test/__init__.py
View file @
40d0a458
...
...
@@ -9,7 +9,7 @@ import codecs
def
create_file
(
cnx
,
data
,
data_name
=
None
,
**
kwargs
):
"""Create a File entity"""
data_name
=
data_name
or
data
.
decode
(
"utf-8"
)
kwargs
.
setdefault
(
"data_format"
,
u
"text/plain"
)
kwargs
.
setdefault
(
"data_format"
,
"text/plain"
)
return
cnx
.
create_entity
(
"File"
,
data
=
Binary
(
data
),
data_name
=
data_name
,
**
kwargs
)
...
...
test/data/cat.py
View file @
40d0a458
import
sys
with
open
(
sys
.
argv
[
1
])
as
f
:
print
(
f
.
read
())
print
(
(
f
.
read
())
)
test/data/reverse.py
View file @
40d0a458
"""Reverse file content given as argument."""
from
__future__
import
print_function
import
sys
with
open
(
sys
.
argv
[
1
])
as
f
:
...
...
test/data/truncate.py
View file @
40d0a458
"""Truncate file given as argument by one character."""
from
__future__
import
print_function
import
sys
with
open
(
sys
.
argv
[
1
])
as
f
:
...
...
test/unittest_entities.py
View file @
40d0a458
"""cubicweb-dataprocessing unit tests for entities"""
from
__future__
import
absolute_import
import
json
import
unittest
...
...
@@ -26,12 +26,12 @@ class TransformationSequenceTC(CubicWebTC):
with
self
.
admin_access
.
repo_cnx
()
as
cnx
:
s1
=
cnx
.
create_entity
(
"TransformationScript"
,
name
=
u
"s1"
,
name
=
"s1"
,
implemented_by
=
create_file
(
cnx
,
b
"pass"
),
)
s2
=
cnx
.
create_entity
(
"TransformationScript"
,
name
=
u
"s2"
,
name
=
"s2"
,
implemented_by
=
create_file
(
cnx
,
b
"pass"
),
)
seq
=
cnx
.
create_entity
(
"TransformationSequence"
)
...
...
@@ -39,7 +39,7 @@ class TransformationSequenceTC(CubicWebTC):
"TransformationStep"
,
index
=
2
,
step_script
=
s1
,
parameters
=
u
'{"foo": "bar"}'
,
parameters
=
'{"foo": "bar"}'
,
in_sequence
=
seq
,
)
cnx
.
create_entity
(
...
...
@@ -50,7 +50,7 @@ class TransformationSequenceTC(CubicWebTC):
)
cnx
.
commit
()
scripts_params
=
proc
.
cw_adapt_to
(
"IDataProcess"
).
process_scripts
expected
=
[(
s2
,
None
),
(
s1
,
u
'{"foo": "bar"}'
)]
expected
=
[(
s2
,
None
),
(
s1
,
'{"foo": "bar"}'
)]
self
.
assertEqual
(
list
(
scripts_params
),
expected
)
...
...
@@ -63,9 +63,9 @@ class DataProcessAdapterTC(CubicWebTC):
cnx
,
"DataTransformationProcess"
,
script
,
parameters
=
u
'{"from": "zorglub"}'
,
parameters
=
'{"from": "zorglub"}'
,
)
inputfile
=
create_file
(
cnx
,
b
"data"
,
data_encoding
=
u
"latin1"
)
inputfile
=
create_file
(
cnx
,
b
"data"
,
data_encoding
=
"latin1"
)
# Trigger "start" transition.
proc
.
cw_set
(
process_input_file
=
inputfile
)
cnx
.
commit
()
...
...
@@ -86,7 +86,7 @@ class DataProcessAdapterTC(CubicWebTC):
proc
=
create_process
(
cnx
,
"DataTransformationProcess"
,
script
,
parameters
=
parameters
)
inputfile
=
create_file
(
cnx
,
b
"data"
,
data_encoding
=
u
"latin1"
)
inputfile
=
create_file
(
cnx
,
b
"data"
,
data_encoding
=
"latin1"
)
# Trigger "start" transition.
proc
.
cw_set
(
process_input_file
=
inputfile
)
with
self
.
assertRaises
(
ValueError
)
as
cm
:
...
...
test/unittest_hooks.py
View file @
40d0a458
"""cubicweb-dataprocessing hooks tests"""
from
__future__
import
absolute_import
import
codecs
...
...
@@ -16,7 +16,7 @@ class DataProcessWorkflowHooksTC(CubicWebTC):
"DataValidationProcess"
:
"validation"
,
}[
process_etype
]
script
=
script_from_code
(
cnx
,
script_kind
,
scriptcode
,
name
=
u
"%s script"
%
process_etype
cnx
,
script_kind
,
scriptcode
,
name
=
"%s script"
%
process_etype
)
process
=
create_process
(
cnx
,
process_etype
,
script
)
cnx
.
commit
()
...
...
@@ -29,7 +29,7 @@ class DataProcessWorkflowHooksTC(CubicWebTC):
def
test_data_process_autostart
(
self
):
with
self
.
admin_access
.
repo_cnx
()
as
cnx
:
script
=
cnx
.
create_entity
(
"ValidationScript"
,
name
=
u
"v"
)
script
=
cnx
.
create_entity
(
"ValidationScript"
,
name
=
"v"
)
create_file
(
cnx
,
b
"1/0"
,
reverse_implemented_by
=
script
)
process
=
create_process
(
cnx
,
"DataValidationProcess"
,
script
)
cnx
.
commit
()
...
...
@@ -83,9 +83,9 @@ class DataProcessWorkflowHooksTC(CubicWebTC):
with
self
.
admin_access
.
repo_cnx
()
as
cnx
:
script
=
cnx
.
create_entity
(
"TransformationScript"
,
name
=
u
"HTML"
,
name
=
"HTML"
,
implemented_by
=
create_file
(
cnx
,
content
.
encode
(
"ascii"
)),
output_format
=
u
"text/html"
,
output_format
=
"text/html"
,
)
cnx
.
commit
()
process
=
create_process
(
cnx
,
"DataTransformationProcess"
,
script
)
...
...
@@ -99,12 +99,12 @@ class DataProcessWorkflowHooksTC(CubicWebTC):
)
rset
=
cnx
.
find
(
"File"
,
produced_by
=
process
)
output
=
rset
.
one
()
self
.
assertEqual
(
output
.
data_format
,
u
"text/html"
)
self
.
assertEqual
(
output
.
data_format
,
"text/html"
)
self
.
assertEqual
(
output
.
read
(),
b
"<p>blah</p>"
)
def
test_data_validation_process_validated_by
(
self
):
with
self
.
admin_access
.
repo_cnx
()
as
cnx
:
script
=
cnx
.
create_entity
(
"ValidationScript"
,
name
=
u
"v"
)
script
=
cnx
.
create_entity
(
"ValidationScript"
,
name
=
"v"
)
create_file
(
cnx
,
b
"pass"
,
reverse_implemented_by
=
script
)
process
=
create_process
(
cnx
,
"DataValidationProcess"
,
script
)
cnx
.
commit
()
...
...
@@ -120,14 +120,14 @@ class DataProcessWorkflowHooksTC(CubicWebTC):
def
test_data_process_dependency
(
self
):
"""Check data processes dependency"""
with
self
.
admin_access
.
repo_cnx
()
as
cnx
:
vscript
=
script_from_code
(
cnx
,
"validation"
,
b
"pass"
,
u
"v"
)
vscript
=
script_from_code
(
cnx
,
"validation"
,
b
"pass"
,
"v"
)
vprocess
=
create_process
(
cnx
,
"DataValidationProcess"
,
vscript
)
cnx
.
commit
()
tscript
=
script_from_code
(
cnx
,
"transformation"
,
b
"import sys; sys.stdout.write(open(sys.argv[1]).read())"
,
u
"t"
,
"t"
,
)
tprocess
=
create_process
(
cnx
,
"DataTransformationProcess"
,
tscript
)
tprocess
.
cw_set
(
process_depends_on
=
vprocess
)
...
...
@@ -148,14 +148,14 @@ class DataProcessWorkflowHooksTC(CubicWebTC):
def
test_data_process_dependency_validation_error
(
self
):
"""Check data processes dependency: validation process error"""
with
self
.
admin_access
.
repo_cnx
()
as
cnx
:
vscript
=
script_from_code
(
cnx
,
"validation"
,
b
"1/0"
,
u
"v"
)
vscript
=
script_from_code
(
cnx
,
"validation"
,
b
"1/0"
,
"v"
)
vprocess
=
create_process
(
cnx
,
"DataValidationProcess"
,
vscript
)
cnx
.
commit
()
tscript
=
script_from_code
(
cnx
,
"transformation"
,
b
"import sys; print(open(sys.argv[1]).read())"
,
u
"t"
,
"t"
,
)
tprocess
=
create_process
(
cnx
,
"DataTransformationProcess"
,
tscript
)
tprocess
.
cw_set
(
process_depends_on
=
vprocess
)
...
...
test/unittest_schema.py
View file @
40d0a458
"""cubicweb-dataprocessing unit tests for schema"""
from
__future__
import
absolute_import
from
six
import
text_type
...
...
@@ -16,10 +16,10 @@ class TransformationSequenceTC(testlib.CubicWebTC):
self
.
file1_eid
=
create_file
(
cnx
,
b
"pass"
).
eid
self
.
file2_eid
=
create_file
(
cnx
,
b
"1/0"
).
eid
self
.
script1_eid
=
cnx
.
create_entity
(
"TransformationScript"
,
name
=
u
"s1"
,
implemented_by
=
self
.
file1_eid
"TransformationScript"
,
name
=
"s1"
,
implemented_by
=
self
.
file1_eid
).
eid
self
.
script2_eid
=
cnx
.
create_entity
(
"TransformationScript"
,
name
=
u
"s2"
,
implemented_by
=
self
.
file2_eid
"TransformationScript"
,
name
=
"s2"
,
implemented_by
=
self
.
file2_eid
).
eid
cnx
.
commit
()
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment