diff --git a/actions/CMakeLists.txt b/actions/CMakeLists.txt index 90f98aa6c1a51439f859649a864205f60ce0b5aa..8c8b88f572cdab4f4da7c473137361c81537fbba 100644 --- a/actions/CMakeLists.txt +++ b/actions/CMakeLists.txt @@ -1,4 +1,5 @@ add_custom_target(actions ALL) +add_subdirectory(doc) add_subdirectory(tests) pm_action_init() diff --git a/actions/doc/CMakeLists.txt b/actions/doc/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4dc86eb219c7893e1cd742181bc351cebc855c39 --- /dev/null +++ b/actions/doc/CMakeLists.txt @@ -0,0 +1,5 @@ +set(ACTION_RST +index_dev.rst +) + +add_doc_source(NAME actions RST ${ACTION_RST}) diff --git a/actions/doc/index_dev.rst b/actions/doc/index_dev.rst new file mode 100644 index 0000000000000000000000000000000000000000..22d681be4872ec598e3ae70f449737b00faaa24a --- /dev/null +++ b/actions/doc/index_dev.rst @@ -0,0 +1,362 @@ +:mod:`test_actions.ActionTestCase` - Testing Actions +================================================================================ + +This module is **not** part of the |project| binary distribution. That is the +productive bit running to produce models. It is only part of the source +distribution intended to help developing |project|. Basically it supports you +creating new actions along immediate tests, which will be stored as unit tests +and stay available to monitor later changes. + +.. note:: + + A couple of different paths will be mentioned in the following. To make + things easier to tell apart, a prefix :file:`<SOURCE>` refers to the code + repository, :file:`<BUILD>` to the build directory tree. + +Inside the development environment, the module is only available to unit tests +in the :file:`<SOURCE>/actions/tests` directory. There is one special thing +about using it in your tests for an action, emerging from the way ``make`` runs +unit tests as set up via |cmake|. |python| modules are imported from the source +directory, here this is :file:`<SOURCE>/actions/tests`, while the tests run +inside :file:`<BUILD>/tests`, here this is :file:`<BUILD>/tests/actions`. When +|python| imports a module, its usually compiled into bytecode. This new file +would clutter up the source repository, it would always show up as untracked +file on ``git status``. To prevent this, tell |python| to stop producing +bytecode right at the beginning of your test-script: + +.. testcode:: nobytecode + :hide: + + import sys + sys.dont_write_bytecode = True + +.. code-block:: python + :emphasize-lines: 5 + :linenos: + + import sys + + # this is needed so there will be no test_actions.pyc created in the source + # directory + sys.dont_write_bytecode = True + +Line 5 does the trick. This needs to be set by you in every action unit test +file since |python| only recognises it **before** the module is imported. +Otherwise a module could disable bytecoding for all other modules loaded. + +Testing actions, basically those are commands run in a shell, is very similar +across various actions. Additionally, there are some things that should be +tested for all actions like exit codes. That is why this module exists. + +When developing an action, you will try it in the shell during the process. You +have to check that its doing what you intend, that it delivers the right +output, that it just behaves right on various kinds of input. This module +supports you by providing functionality to run scripts out of |python|. The +goal is to not trigger test runs manually in a shell but have a script that +does it for you. From there, you do not need to remember all the calls you +punched into the command line a year ago, when you come back to change +something, add new functionality, etc.. + +-------------------------------------------------------------------------------- +Creating an Action Unit Test Script +-------------------------------------------------------------------------------- +In the next couple of paragraphs, we will walk through setting up a new unit +test script for an imaginary action. We will continuously extend the file +started above, so keep an eye on line numbers. Lets just assume your action is +called ``do-awesome`` for the rest of this section. + +The Test Script +-------------------------------------------------------------------------------- +The script to supervise your action needs to be placed in +:file:`<SOURCE>/actions/tests` and follow the naming convention +:file:`test_action_<NAME>.py`, where :file:`<NAME>` is the name for your +action. So here we create a file :file:`test_action_do_awesome.py` (recognise +the underscore between ``do`` and ``awesome`` instead of a hyphen, that's +|pep8|_). + +.. code-block:: console + + $ touch <SOURCE>/actions/tests/test_action_do_awesome.py + $ + +As a starter, we disable bytecode compilation in the script: + +.. testsetup:: actiontest + :hide: + + import sys + sys.dont_write_bytecode = True + +.. code-block:: python + :linenos: + + import sys + + # this is needed so there will be no test_actions.pyc created in the source + # directory + sys.dont_write_bytecode = True + +|cmake| Integration +-------------------------------------------------------------------------------- +As always, when introducing new material to |project|, it has to be announced +to the |cmake| build system. For action unit tests, fire up +:file:`<SOURCE>/actions/tests/CMakeLists.txt` in your favourite text editor and +add your new script: + +.. code-block:: cmake + :emphasize-lines: 3 + :linenos: + + set(ACTION_UNIT_TESTS + test_action_help.py + test_action_do_awesome.py + test_actions.py # leave this as last item so it will be executed first! + ) + + promod3_unittest(MODULE actions SOURCES "${ACTION_UNIT_TESTS}" TARGET actions) + +The important thing is to leave :file:`test_actions.py` as last item in the +list. This script contains the tests around the +:class:`test_actions.ActionTestCase` class, which is the foundation of the +tests for your action. If this class is broken, we are lost. Putting it as the +last element in the list, |cmake| will execute this script first, before any +other action test script is run. + +Creating a Test Subclass +-------------------------------------------------------------------------------- +:class:`test_actions.ActionTestCase` is sort of a template class for your +tests. By spawning off from this you inherit a bunch of useful methods for your +testing. To make it work, the childclass needs to be set up properly. But +first, :file:`test_actions.py` has to be loaded as a module: + +.. testcode:: actiontest + :hide: + + sys.path.insert(0, __actiontest_path__) + import test_actions + +.. code-block:: python + :linenos: + :lineno-start: 6 + + import test_actions + +Now create the childclass for your action. Go for :class:`<NAME>ActionTests` as +a naming scheme: + +.. testcode:: actiontest + :hide: + + class DoAwesomeActionTests(test_actions.ActionTestCase): + def __init__(self, *args, **kwargs): + test_actions.ActionTestCase.__init__(self, *args, **kwargs) + self.pm_action = 'do-awesome' + +.. code-block:: python + :linenos: + :lineno-start: 7 + + class DoAwesomeActionTests(test_actions.ActionTestCase): + def __init__(self, *args, **kwargs): + test_actions.ActionTestCase.__init__(self, *args, **kwargs) + self.pm_action = 'do-awesome' + +Pay attention that in your own class, you must set :attr:`pm_action` to make +everything work. Also :meth:`__init__` needs certain parameters, as everything +is derived from the :class:`unittest.TestCase` class. + +Must Have Tests +-------------------------------------------------------------------------------- +What needs testing without exclusion are the exit codes of actions. Those +states will be placed in the userlevel documentation. This topic is already +covered in :class:`test_actions.ActionTestCase` by :meth:`RunExitStatusTest`. +As an example, testing for ``$?=0`` could work like this: + +.. testcode:: actiontest + :hide: + + class DoAwesomeActionTests(test_actions.ActionTestCase): + def __init__(self, *args, **kwargs): + test_actions.ActionTestCase.__init__(self, *args, **kwargs) + self.pm_action = 'do-awesome' + + def testExit0(self): + self.RunExitStatusTest(0, list()) + +.. code-block:: python + :linenos: + :lineno-start: 11 + + def testExit0(self): + self.RunExitStatusTest(0, list()) + +That will call the action stored in :attr:`pm_action` with the provided list of +parameters and check that ``0`` is returned on the command line. + +In a more general way, you need to test that your action is working as +intended. Do not forget some negative testing, with the idea in mind what +happens if a user throws dirty input data in. + +Making the Script Executable +-------------------------------------------------------------------------------- +In |project|, unit tests are run via |ost_s|_ and |python|'s +:class:`unittest.TestCase`. Those are called when the test module is executed +as a script: + +.. testcode:: actiontest + :hide: + + import unittest + + class DoAwesomeActionTests(test_actions.ActionTestCase): + def __init__(self, *args, **kwargs): + test_actions.ActionTestCase.__init__(self, *args, **kwargs) + self.pm_bin = os.path.join(os.getcwd(), os.pardir, 'stage', 'bin', + 'pm') + self.pm_action = 'help' + + def testExit0(self): + self.RunExitStatusTest(0, list()) + + if __name__ == "__builtin__": + import os + suite = unittest.TestLoader().loadTestsFromTestCase(DoAwesomeActionTests) + unittest.TextTestRunner().run(suite) + +.. code-block:: python + :linenos: + :lineno-start: 13 + + if __name__ == "__main__": + from ost import testutils + testutils.RunTests() + +These three lines should be the same for all unit tests. + +Running the Test Script +-------------------------------------------------------------------------------- +Unit tests are executed via ``make check`` and so are |project| action tests. +But for every test script, we also provide a private ``make`` target, ending +with :file:`_run`. To solely run the tests for the awesome action, hit + +.. code-block:: console + + $ make test_action_do_awesome.py_run + +Output of :class:`test_actions.ActionTestCase` +-------------------------------------------------------------------------------- +When running the test script you will notice that its not really talkative. +Basically you do not see output to :file:`stdout`/ :file:`stderr` of your +action, while the test script fires it a couple of times. That is by design. +When running the full unit test suite, usually nobody wants to see the output +of **everything** tested and working. The interesting bits are where we fail. +But for developing a new application you certainly need all the output you can +get. For this, some functions in :class:`test_actions.ActionTestCase` have a +parameter :attr:`verbose`. That triggers specific functions to flush captured +output onto the command line. The idea is to turn it on for development, but +once done, disable it to keep output of unit tests low. + +To get the test for exit code ``0`` talking to you, just do + +.. testcode:: actiontest + :hide: + + import unittest + + class DoAwesomeActionTests(test_actions.ActionTestCase): + def __init__(self, *args, **kwargs): + test_actions.ActionTestCase.__init__(self, *args, **kwargs) + self.pm_bin = os.path.join(os.getcwd(), os.pardir, 'stage', 'bin', + 'pm') + self.pm_action = 'help' + + def testExit0(self): + self.RunExitStatusTest(0, list(), verbose=True) + + if __name__ == "__builtin__": + import os + suite = unittest.TestLoader().loadTestsFromTestCase(DoAwesomeActionTests) + unittest.TextTestRunner().run(suite) + +.. testoutput:: actiontest + :hide: + :options: +NORMALIZE_WHITESPACE +ELLIPSIS + + stdout of '.../doc/../stage/bin/pm help' + ------ + Following actions are available: + build-rawmodel + help + Each action should respond to "--help". + ------ + stderr of '.../doc/../stage/bin/pm help' + ------ + ------ + +.. code-block:: python + :linenos: + :lineno-start: 11 + + def testExit0(self): + self.RunExitStatusTest(0, list(), verbose=True) + +and + +.. testcode:: actiontest + :hide: + + import unittest + + class DoAwesomeActionTests(test_actions.ActionTestCase): + def __init__(self, *args, **kwargs): + test_actions.ActionTestCase.__init__(self, *args, **kwargs) + self.pm_bin = os.path.join(os.getcwd(), os.pardir, 'stage', 'bin', + 'pm') + self.pm_action = 'help' + + def testExit0(self): + self.RunExitStatusTest(0, list()) + + if __name__ == "__builtin__": + import os + suite = unittest.TestLoader().loadTestsFromTestCase(DoAwesomeActionTests) + unittest.TextTestRunner().run(suite) + +.. code-block:: python + :linenos: + :lineno-start: 11 + + def testExit0(self): + self.RunExitStatusTest(0, list()) + +keeps it silent (:attr:`verbose` is set to ``False`` by default). If enabled, +output will be separated into :file:`stdout` and :file:`stderr`: + +.. code-block:: console + + $ make test_action_do_awesome.py_run + <Lots of output from the build process> + [100%] running checks test_action_do_awesome.py + stdout of '<BUILD>/stage/bin/pm do-awesome' + ------ + <Output to stdout> + ------ + stderr of '<BUILD>/stage/bin/pm do-awesome' + ------ + <Output to stderr> + ------ + +-------------------------------------------------------------------------------- +Unit Test Actions API +-------------------------------------------------------------------------------- + +.. autoclass:: test_actions.ActionTestCase + :members: + +.. LocalWords: ActionTestCase currentmodule cmake bytecode emphasize sys py +.. LocalWords: linenos pyc dont promod unittest childclass lineno init args +.. LocalWords: ActionTests DoAwesomeActionTests kwargs attr meth TestCase +.. LocalWords: userlevel RunExitStatusTest testExit ost testutils RunTests +.. LocalWords: stdout stderr testcode nobytecode testsetup actiontest os +.. LocalWords: getcwd pardir builtin testoutput NORMALIZE WHITESPACE API +.. LocalWords: rawmodel autoclass diff --git a/actions/tests/CMakeLists.txt b/actions/tests/CMakeLists.txt index 64930f89fb9d6bdd8db67bca6d1ed3b4b59db7ad..159b8d2830e4311e24ca407af2d20d125c5dcd6f 100644 --- a/actions/tests/CMakeLists.txt +++ b/actions/tests/CMakeLists.txt @@ -3,4 +3,9 @@ set(ACTION_UNIT_TESTS test_actions.py # leave this as last item so it will be executed first! ) -promod3_unittest(MODULE actions SOURCES "${ACTION_UNIT_TESTS}" TARGET actions) \ No newline at end of file +promod3_unittest(MODULE actions SOURCES "${ACTION_UNIT_TESTS}" TARGET actions) + +if(NOT DISABLE_DOCUMENTATION) + add_doc_dependency(NAME actions + DEP "${CMAKE_CURRENT_SOURCE_DIR}/test_actions.py") +endif() \ No newline at end of file diff --git a/actions/tests/test_actions.py b/actions/tests/test_actions.py index 9030f24afdf4c17f45504d27137c7722d3b18bab..94eb6a6bd356f33f6cb0f0ad6f133626db697b92 100644 --- a/actions/tests/test_actions.py +++ b/actions/tests/test_actions.py @@ -10,10 +10,33 @@ import ost ost.PushVerbosityLevel(2) class ActionTestCase(unittest.TestCase): + """ + Class to help developing actions. Comes with a lot of convenience wrappers + around what should be tested and serves as a recorder for test calls... + just for in two years when you come back to rewrite the whole action... + + While inheriting this class, :attr:`pm_action` needs to be defined. + Otherwise the whole idea does not work. + + .. attribute:: pm_bin + + This is the path of the ``pm`` binary. Automatically set by calling + :meth:`~ActionTestCase.__init__` inside the initialisation of your class. + + :type: :class:`str` + + .. attribute:: pm_action + + The action to be tested. Needs to be set by your initialisation routine, + **after** calling :meth:`~ActionTestCase.__init__` from here. Skip the + "pm-" in front of the action name. + + :type: :class:`str` + """ def __init__(self, *args, **kwargs): - ''' + """ Convenience stuff for action testing. - ''' + """ # Determining the pm binary to be called. Going hard-coded is a bad # thing. But this is a unit test and we now where we are. Also be # putting it into the 'setUp' function, we only need to change it once, @@ -87,6 +110,11 @@ class ActionTestCase(unittest.TestCase): "but returned as '%d'." % exit_code_run) def testPMExists(self): + """ + This is an internal test, executed when the source code of the test + class is run as unit test. Verifies that :attr:`pm_bin` is an existing + file (also complains if a directory is found instead). + """ self.assertEqual(os.path.isfile(self.pm_bin), True, msg="Could not find 'pm' bin at '%s', " % self.pm_bin+ "cannot proceed unit tests.") @@ -96,3 +124,5 @@ if __name__ == "__main__": from ost import testutils sys.dont_write_bytecode = True testutils.RunTests() + +# LocalWords: attr meth ActionTestCase init str stdout stderr param bool diff --git a/cmake_support/PROMOD3.cmake b/cmake_support/PROMOD3.cmake index ce2b285a78c60711a65ce00d2bf53decb9b9210f..e3456820e8219c6c80114b5648e111beb519dccb 100644 --- a/cmake_support/PROMOD3.cmake +++ b/cmake_support/PROMOD3.cmake @@ -999,13 +999,7 @@ macro(setup_boost) endmacro(setup_boost) #------------------------------------------------------------------------------- -# Synopsis: -# add_doc_dependency(NAME module DEP depending module) -# -# Description: -# Add a dependency for the doc build system. -# NAME - name of the module, these dependencies belong to -# DEP - modules to be added +# Documentation to be found in cmake_support/doc/index.rst #------------------------------------------------------------------------------- macro(add_doc_dependency) parse_argument_list(_ADD_ARG "NAME;DEP" "" ${ARGN}) @@ -1033,13 +1027,7 @@ macro(add_doc_dependency) endmacro(add_doc_dependency) #------------------------------------------------------------------------------- -# Synopsis: -# add_doc_source(NAME module RST rst1 rst2) -# -# Description: -# Add reStructuredText sources for the doc build system. -# NAME - name of the module, the rst files belong to -# RST - file/ cmake list of rst files to be added +# Documentation to be found in cmake_support/doc/index.rst #------------------------------------------------------------------------------- macro(add_doc_source) parse_argument_list(_ARG "NAME;RST" "" ${ARGN}) diff --git a/cmake_support/doc/index.rst b/cmake_support/doc/index.rst index 368f4b1c8656de76896f27a6ff0e509add08ae03..a84784420b26b119b89cba4adee7411d30a837cf 100644 --- a/cmake_support/doc/index.rst +++ b/cmake_support/doc/index.rst @@ -57,7 +57,7 @@ Unit Tests needs to be a single word. Ends up in ``make help`` as a prefix, nothing will break if it does not match the name of any existing module. - ``Sources`` + ``SOURCES`` Describe a set of files hosting unit test code here. If its a wild mix of |C++| and |python| files does not matter, |cmake| will sort this out for you. But the programming language makes a difference for the ``make`` @@ -79,7 +79,59 @@ Unit Tests build directory. ``TARGET`` - This defines an additional dependency for the unit test. + This defines an additional dependency for the unit test. That is, before + running this unit test, this target will be built. + +Documentation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. cmake:command:: add_doc_source + + .. code-block:: cmake + + add_doc_source(NAME name + RST rst1 [rst2...]) + + Add reStructuredText sources for the doc build system. This is most preferable + used in :file:`doc` directories for keeping the documentation sorted per + module. This does not create any ``make`` targets. Lists filled here will all + be evaluated in the :file:`doc/CMakeLists.txt` of the repository root. + + The parameters are: + + ``NAME`` + Specify the name of the module this branch of documentation belongs to. + Needs to be set, needs to be a single word. Using module names is best + practice, while nothing will break if it does not refer to an existing one. + You will find a directory in :file:`doc/source` with that name in the build + root. + + ``RST`` + Describe a set of files containing the documentation. Feed it a single file + name or a |cmake| list. + +.. cmake:command:: add_doc_dependency + + .. code-block:: cmake + + add_doc_source(NAME name + DEP dependency1 [dependency2...]) + + Add a dependency to the doc build system. For an existing name, add some + dependencies when it comes to building documentation. Mostly for internal use. + + The parameters are: + + ``NAME`` + Specify a name the dependencies belong to. This name needs to be already + known in the doc build system. Names of |python| modules are good, otherwise + names introduced by :cmake:command:`add_doc_source` work well. Dependencies + will be create for all reStructuredText files listed by + :cmake:command:`add_doc_source` under this name and for all ``make`` + targets related to the documentation. + + ``DEP`` + Hand over a dependency here or a |cmake| list. Files work, if given with + absolute path. Actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -113,4 +165,4 @@ Actions .. ----------------- .. LocalWords: cmake PROMOD CMakeLists txt promod unittest codetest xml py -.. LocalWords: libexec +.. LocalWords: libexec reStructuredText RST subtree rst DEP diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 6517753a1aecd7195c3a584d9976e1722315edcb..a922908435f7092c5dc4de3ab3ca5942ab9b1665 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -31,7 +31,8 @@ set(_SPHINX_CONF_SUBST_DICT PROMOD3_VERSION_MAJOR="${PROMOD3_VERSION_MAJOR}" OST_PYMOD_PATH="${OST_PYMOD_PATH}" OST_DOC_URL="${OST_DOC_URL}" LIB_DIR="${LIB_DIR}" - THIS_DIR="${_RST_SOURCE_DIR}") + THIS_DIR="${_RST_SOURCE_DIR}" + PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}") set(_CONF_SUBST_DICT -DINPUT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in) list(APPEND _CONF_SUBST_DICT -DOUT_FILE=${_SPHINX_CONF_PY}) diff --git a/doc/conf.py.in b/doc/conf.py.in index 940e8c09740e58ac3ecf550b16e5ea4942ef883d..7d887449d9c74677fd4db385b00274b69eba0b1f 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -24,6 +24,7 @@ import sys sys.path.insert(0, r'@LIB_STAGE_PATH@/@PYTHON_MODULE_PATH@') sys.path.insert(1, r'@OST_PYMOD_PATH@') sys.path.insert(2, r'@THIS_DIR@') +sys.path.insert(3, r'@PROJECT_SOURCE_DIR@/actions/tests/') # -- General configuration ----------------------------------------------------- @@ -283,6 +284,8 @@ rst_epilog = """ .. |C++| replace:: C++ .. _ost_s: http://www.OpenStructure.org .. _nameattr: @PYTHON_DOC_URL@/library/__main__.html +.. |pep8| replace:: PEP 8 +.. _pep8: https://www.python.org/dev/peps/pep-0008/ """ % project # in some versions of sphinx, doctest invokes doctest_path AFTER executing code doctest_global_setup = """ @@ -292,6 +295,9 @@ sys.path.insert(1, '@OST_PYMOD_PATH@') import ost +# this one is needed for actiontest doctests +__actiontest_path__ = '@PROJECT_SOURCE_DIR@/actions/tests/' + # We define a LogSink here, pushing everything to stdout so doctest recognises # it. class DocTestLogger(ost.LogSink): diff --git a/doc/developers.rst b/doc/developers.rst index a750240d77b63b5c476fcc12b03d2c08874e425d..84b8f7022d642da56710f574abb8139f6cd15ce2 100644 --- a/doc/developers.rst +++ b/doc/developers.rst @@ -12,6 +12,7 @@ Contents: core/setcompoundschemlib core/index rawmodel/index + actions/index_dev buildsystem contributing cmake/index