|
|
(15 intermediate revisions by 5 users not shown) |
Line 5: |
Line 5: |
| 2. Install Behat's dependencies: | | 2. Install Behat's dependencies: |
| <source lang="bash" enclose="div"> | | <source lang="bash" enclose="div"> |
− | $ sudo apt-get install curl openjdk-7-jre-headless | + | $ sudo apt-get install curl openjdk-8-jre-headless |
| </source> | | </source> |
| | | |
Line 28: |
Line 28: |
| </source> | | </source> |
| | | |
− | 5. Make the directory and any subdirectories writeable by Mahara (check this is correct): | + | 5. Make the directory and any subdirectories writeable by Mahara (check the directory is correct): |
| <source lang="bash" enclose="div"> | | <source lang="bash" enclose="div"> |
| $ sudo chmod 777 -R /var/lib/maharadata/master_behat | | $ sudo chmod 777 -R /var/lib/maharadata/master_behat |
| </source> | | </source> |
− | | + | <span id="chown"></span> |
− | 6. Change the directory and any subdirectories permissions to be owned by apache user (check this is correct): | + | 6. Change the directory and any subdirectories permissions to be owned by apache user (check the directory is correct): |
| <source lang="bash" enclose="div"> | | <source lang="bash" enclose="div"> |
− | $ sudo chown -R www-data.www-data /var/lib/maharadata/master_behat | + | $ sudo chown -R www-data:www-data /var/lib/maharadata/master_behat |
| </source> | | </source> |
| | | |
Line 71: |
Line 71: |
| [[Category:Behat]] | | [[Category:Behat]] |
| | | |
− | == How to Run Mahara behat test on Chrome == | + | == Running Behat tests with html report and screenshots on failed steps == |
| + | |
| + | To run tests with html output, add the flag 'html' to your test run command, e.g: |
| + | |
| + | <source>./test/behat/mahara_behat.sh run create_page.feature html</source> |
| + | |
| + | This will automatically open an html report of the test run in your browser. If there were failed steps, the the html report plugin takes a screenshot of the page that failed. |
| + | You can find the screenshots in your behat dataroot (usually /var/lib/maharadata/master_behat) at the following path: |
| + | behat/html_results/<feature title>/<scenario title>.png |
| + | |
| + | Hopefully we will rig up the report to link to the screenshot soon. |
| + | |
| + | == How to Run Mahara behat test on Chrome for versions older than 17.04== |
| | | |
| {{note|Mahara behat is running on chrome driver by default in 17:04+}} | | {{note|Mahara behat is running on chrome driver by default in 17:04+}} |
Line 98: |
Line 110: |
| for example ./test/behat/mahara_behat.sh run example.feature | | for example ./test/behat/mahara_behat.sh run example.feature |
| | | |
− | == Steps for creating a html report and a screenshot for behat tests''' == | + | == Debugging == |
− | | |
− | === 1- Install BehatHtmlFormatterPlugin ===
| |
− | | |
− | This is from https://github.com/dutchiexl/BehatHtmlFormatterPlugin.
| |
− | Open your /mahara/external/composer.json and add the following line in the "require" block:
| |
− | "emuse/behat-html-formatter": ">=0.1",
| |
− | ''(you don't need the trailing comma if it's the last requirement.)''
| |
− | | |
− | Then navigate to your code checkout in a terminal and run
| |
− | make initcomposer
| |
− | | |
− | Check the /mahara/external/vendor folder and you should see a subfolder called emuse. If not, look for and try to resolve any composer errors.
| |
− | | |
− | === 2- Edit your behat.yml to use the html formatter ===
| |
− | | |
− | You'll find the behat.yml file in your behat dataroot. If you're not sure where that is, look in your config.php file (/mahara/htdocs/config.php).
| |
− | In my case it says
| |
− | $cfg->behat_dataroot = "/var/lib/maharadata/master_behat";
| |
− | So, my behat.yml will be in var/lib/maharadata/master_behat/behat/behat.yml
| |
− | | |
− | You'll need to add two things to the file. Under formatters add the html output path for the report like this:
| |
− | formatters:
| |
− | html:
| |
− | output_path: %paths.base%/results/html/
| |
− | So after a test run your html report will be found here: ''your behat_dataroot''/behat/results/html/''my_report''.html '''''(NB: I'm thinking about making this simpler so there are less nested folders.)'''''
| |
− | | |
− | Then, under extensions, add the following settings for the HTML report, making sure the line beginning 'emuse...' is indented to the same level as Behat\MinkExtension.
| |
− | extensions:
| |
− | emuse\BehatHTMLFormatter\BehatHTMLFormatterExtension:
| |
− | name: html
| |
− | renderer: Twig,Behat2
| |
− | file_name: index
| |
− | print_args: true
| |
− | print_outp: true
| |
− | loop_break: true
| |
− | Here the filename is index, meaning that the report file will be called index.html and overwritten after every test run.
| |
− | If you leave out the file_name line here, the default will prevail, which is that every test run will produce two separate reports, Twig_''date/time'' and Behat2_''datetime''.
| |
− | '''''(NB: Useful to look into producing a combined report with a unique name for each test run?)'''''
| |
− | | |
− | It should look like the below code(copied my yml file for reference)
| |
− | | |
− | default:
| |
− | autoload:
| |
− | - /home/niranjanbandi/code/mahara/htdocs/testing/frameworks/behat/classes
| |
− | formatters:
| |
− | html:
| |
− | output_path: %paths.base%/results/html/
| |
− | extensions:
| |
− | Behat\MinkExtension:
| |
− | base_url: 'http://localhost:8000'
| |
− | files_path: /home/niranjanbandi/code/mahara/test/behat/upload_files
| |
− | javascript_session: selenium2
| |
− | selenium2:
| |
− | browser: chrome
| |
− | goutte: ~
| |
− | emuse\BehatHTMLFormatter\BehatHTMLFormatterExtension:
| |
− | name: html
| |
− | renderer: Twig,Behat2
| |
− | file_name: index
| |
− | print_args: true
| |
− | print_outp: true
| |
− | loop_break: true
| |
− | suites:
| |
− | core_features:
| |
− | paths:
| |
− | - /home/loveshjain/code/mahara/test/behat/features
| |
− | contexts:
| |
− | - BehatMaharaCoreContext
| |
− | - BehatHooks
| |
− | - BehatGeneral
| |
− | - BehatNavigation
| |
− | - BehatView
| |
− | - BehatDataGenerators
| |
− | - BehatAccount
| |
− | - BehatAdmin
| |
− | - BehatForms
| |
− | filters:
| |
− | tags: '@core'
| |
− | | |
− | 3- Change code/mahara/test/behat/mahar_behat.sh
| |
− | | |
− | you can copy paste the below code with the existing code
| |
− | | |
− | #!/bin/bash
| |
− | | |
− | # Get action and Mahara dir
| |
− | ACTION=$1
| |
− | SCRIPTPATH=`readlink -f "${BASH_SOURCE[0]}"`
| |
− | MAHARAROOT=`dirname $( dirname $( dirname "$SCRIPTPATH" ))`
| |
− | SERVER=0
| |
− | test -z $SELENIUM_PORT && export SELENIUM_PORT=4400
| |
− | test -z $PHP_PORT && export PHP_PORT=8000
| |
− | test -z $XVFB_PORT && export XVFB_PORT=10
| |
− | | |
− | echo "S: $SELENIUM_PORT"
| |
− | echo "P: $PHP_PORT"
| |
− | | |
− | # Wait and check if the selenium server is running in maximum 15 seconds
| |
− | function is_selenium_running {
| |
− | for i in `seq 1 15`; do
| |
− | sleep 1
| |
− | res=$(curl -o /dev/null --silent --write-out '%{http_code}\n' http://localhost:${SELENIUM_PORT}/wd/hub/status)
| |
− | if [ $res == "200" ]; then
| |
− | return 0;
| |
− | fi
| |
− | done
| |
− | return 1;
| |
− | }
| |
− | | |
− | function cleanup {
| |
− | echo "Shutdown Selenium"
| |
− | curl -o /dev/null --silent http://localhost:${SELENIUM_PORT}/selenium-server/driver/?cmd=shutDownSeleniumServer
| |
− | | |
− | if [[ $SERVER ]]
| |
− | then
| |
− | echo "Shutdown PHP server"
| |
− | kill $SERVER
| |
− | fi
| |
− | | |
− | if [[ $1 ]]
| |
− | then
| |
− | exit $1
| |
− | else
| |
− | exit 255
| |
− | fi
| |
− | | |
− | echo "Disable behat test environment"
| |
− | php htdocs/testing/frameworks/behat/cli/util.php -d
| |
− | }
| |
− | | |
− | # Check we are not running as root for some weird reason
| |
− | if [[ "$USER" = "root" ]]
| |
− | then
| |
− | echo "This script should not be run as root"
| |
− | exit 1
| |
− | fi
| |
− | | |
− | cd $MAHARAROOT
| |
− | | |
− | # Trap errors so we can cleanup
| |
− | trap cleanup ERR
| |
− | trap cleanup INT
| |
− | | |
− | if [ "$ACTION" = "action" ]
| |
− | then
| |
− | | |
− | # Wrap the util.php script
| |
− | | |
− | PERFORM=$2
| |
− | php htdocs/testing/frameworks/behat/cli/util.php --$PERFORM
| |
− | | |
− | elif [ "$ACTION" = "run" -o "$ACTION" = "runheadless" -o "$ACTION" = "rundebug" -o "$ACTION" = "runfresh" -o $ACTION = 'rundebugheadless' ]
| |
− | then
| |
− | | |
− | if [[ $2 == @* ]]; then
| |
− | TAGS=$2
| |
− | echo "Only run tests with the tag: $TAGS"
| |
− | elif [ $2 ]; then
| |
− | if [[ $2 == */* ]]; then
| |
− | FEATURE="test/behat/features/$2"
| |
− | else
| |
− | FEATURE=`find test/behat/features -name $2 | head -n 1`
| |
− | fi
| |
− | echo "Only run tests in file: $FEATURE"
| |
− | else
| |
− | echo "Run all tests"
| |
− | fi
| |
− | | |
− | if [ "$ACTION" = "runfresh" ]
| |
− | then
| |
− | echo "Drop the old test site if exist"
| |
− | php htdocs/testing/frameworks/behat/cli/util.php --drop
| |
− | fi
| |
− | | |
− | # Initialise the test site for behat (database, dataroot, behat yml config)
| |
− | php htdocs/testing/frameworks/behat/cli/init.php
| |
− | | |
− | # Run the Behat tests themselves (after any intial setup)
| |
− | if is_selenium_running; then
| |
− | echo "Selenium is running"
| |
− | else
| |
− | echo "Start Selenium..."
| |
− | | |
− | SELENIUM_VERSION_MAJOR=2.53
| |
− | SELENIUM_VERSION_MINOR=1
| |
− | | |
− | SELENIUM_FILENAME=selenium-server-standalone-$SELENIUM_VERSION_MAJOR.$SELENIUM_VERSION_MINOR.jar
| |
− | SELENIUM_PATH=./test/behat/$SELENIUM_FILENAME
| |
− | | |
− | # If no Selenium installed, download it
| |
− | if [ ! -f $SELENIUM_PATH ]; then
| |
− | echo "Downloading Selenium..."
| |
− | wget -q -O $SELENIUM_PATH http://selenium-release.storage.googleapis.com/$SELENIUM_VERSION_MAJOR/$SELENIUM_FILENAME
| |
− | echo "Downloaded"
| |
− | fi
| |
− | | |
− | if [ $ACTION = 'runheadless' -o $ACTION = 'rundebugheadless' ]
| |
− | then
| |
− | # we want to run selenium headless on a different display - this allows for that ;)
| |
− | echo "Starting Xvfb ..."
| |
− | Xvfb :${XVFB_PORT} -ac > /tmp/xvfb.log 2>&1 & echo "PID [$!]"
| |
− | | |
− | DISPLAY=:${XVFB_PORT} nohup java -jar $SELENIUM_PATH -port ${SELENIUM_PORT} > /tmp/selenium.log 2>&1 & echo $!
| |
− | else
| |
− | java -jar $SELENIUM_PATH -port ${SELENIUM_PORT} &> /tmp/selenium.log &
| |
− | fi
| |
− | | |
− | if is_selenium_running; then
| |
− | echo "Selenium started"
| |
− | echo "Selenium started lovesh"
| |
− | else
| |
− | echo "Selenium can't be started"
| |
− | exit 1
| |
− | fi
| |
− | fi
| |
− | | |
− | echo "Start PHP server"
| |
− | php --server localhost:${PHP_PORT} --docroot $MAHARAROOT/htdocs &> /tmp/php.log &
| |
− | SERVER=$!
| |
− | | |
− | BEHATCONFIGFILE=`php htdocs/testing/frameworks/behat/cli/util.php --config`
| |
− | echo "Run Behat..."
| |
− | | |
− | | |
− | OPTIONS=''
| |
− | if [ $ACTION = 'rundebug' -o $ACTION = 'rundebugheadless' ]
| |
− | then
| |
− | OPTIONS=$OPTIONS"--format=pretty --format=html"
| |
− | else
| |
− | OPTIONS=$OPTIONS" --format=pretty --format=html"
| |
− | fi
| |
− | | |
− | if [ "$TAGS" ]; then
| |
− | OPTIONS=$OPTIONS" --tags "$TAGS
| |
− | elif [ "$FEATURE" ]; then
| |
− | OPTIONS=$OPTIONS" "$FEATURE
| |
− | fi
| |
− | | |
− | echo
| |
− | echo "=================================================="
| |
− | echo
| |
− | | |
− | echo ./external/vendor/bin/behat --config $BEHATCONFIGFILE $OPTIONS
| |
− | ./external/vendor/bin/behat --config $BEHATCONFIGFILE $OPTIONS
| |
− | | |
− | echo
| |
− | echo "=================================================="
| |
− | echo
| |
− | echo "Shutdown"
| |
− | cleanup 0
| |
− | else
| |
− | # Help text if we got an unexpected (or empty) first param
| |
− | echo "Expected something like one of the following:"
| |
− | echo
| |
− | echo "# Run all tests:"
| |
− | echo "mahara_behat run"
| |
− | echo ""
| |
− | echo "# Run tests in file \"example.feature\""
| |
− | echo "mahara_behat run example.feature"
| |
− | echo ""
| |
− | echo "# Run tests with specific tag:"
| |
− | echo "mahara_behat run @tagname"
| |
− | echo ""
| |
− | echo "# Run tests with extra debug output:"
| |
− | echo "mahara_behat rundebug"
| |
− | echo "mahara_behat rundebug example.feature"
| |
− | echo "mahara_behat rundebug @tagname"
| |
− | echo ""
| |
− | echo "# Run in headless mode (requires xvfb):"
| |
− | echo "mahara_behat runheadless"
| |
− | echo ""
| |
− | echo "# Run in headless mode with extra debug output:"
| |
− | echo "mahara_behat rundebugheadless"
| |
− | echo ""
| |
− | echo "# Enable test site:"
| |
− | echo "mahara_behat action enable"
| |
− | echo ""
| |
− | echo "# Disable test site:"
| |
− | echo "mahara_behat action disable"
| |
− | echo ""
| |
− | echo "# List other actions you can perform:"
| |
− | echo "mahara_behat action help"
| |
− | exit 1
| |
− | fi
| |
− | | |
− | Step 4 :- Change Behat hooks for getting screenshots :-
| |
− | | |
− | | |
− | <?php
| |
− | /**
| |
− | * @package mahara
| |
− | * @subpackage test/behat
| |
− | * @author Son Nguyen, Catalyst IT Ltd
| |
− | * @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
| |
− | * @copyright For copyright information on Mahara, please see the README file distributed with this software.
| |
− | * @copyright portions from mahara Behat, 2013 David Monllaó
| |
− | *
| |
− | */
| |
− | | |
− | /**
| |
− | * Behat accepts hooks after and before each
| |
− | * suite, feature, scenario and step.
| |
− | *
| |
− | * This methods are used by Behat CLI command.
| |
− | *
| |
− | */
| |
− | | |
− | | |
− | require_once(__DIR__ . '/BehatBase.php');
| |
− | | |
− | use Behat\Behat\Event\SuiteEvent as SuiteEvent,
| |
− | Behat\Behat\Event\FeatureEvent as FeatureEvent,
| |
− | Behat\Behat\Event\ScenarioEvent as ScenarioEvent,
| |
− | Behat\Behat\Event\StepEvent as StepEvent,
| |
− | Behat\Mink\Exception\DriverException as DriverException,
| |
− | WebDriver\Exception\NoSuchWindow as NoSuchWindow,
| |
− | WebDriver\Exception\UnexpectedAlertOpen as UnexpectedAlertOpen,
| |
− | WebDriver\Exception\UnknownError as UnknownError,
| |
− | WebDriver\Exception\CurlExec as CurlExec,
| |
− | WebDriver\Exception\NoAlertOpenError as NoAlertOpenError;
| |
− | | |
− | use Behat\Behat\Hook\Scope\BeforeStepScope;
| |
− | use Behat\Behat\Hook\Scope\AfterStepScope;
| |
− | /**
| |
− | * Hooks to the behat process.
| |
− | *
| |
− | * Implement hooks after and before each
| |
− | * suite, feature, scenario and step for Mahara
| |
− | *
| |
− | * Throws generic Exception because they are captured by Behat.
| |
− | *
| |
− | */
| |
− | class BehatHooks extends BehatBase {
| |
− | | |
− | /**
| |
− | * @var For actions that should only run once.
| |
− | */
| |
− | protected static $initprocessesfinished = false;
| |
− | | |
− | /**
| |
− | * Some exceptions can only be caught in a before or after step hook,
| |
− | * they can not be thrown there as they will provoke a framework level
| |
− | * failure, but we can store them here to fail the step in i_look_for_exceptions()
| |
− | * which result will be parsed by the framework as the last step result.
| |
− | *
| |
− | * @var Null or the exception last step throw in the before or after hook.
| |
− | */
| |
− | protected static $currentstepexception = null;
| |
− | | |
− | /**
| |
− | * If we are saving any kind of dump on failure we should use the same parent dir during a run.
| |
− | *
| |
− | * @var The parent dir name
| |
− | */
| |
− | protected static $faildumpdirname = false;
| |
− | | |
− | /**
| |
− | * Make sure the test site is installed and enabled for behat tests.
| |
− | *
| |
− | * @static
| |
− | * @throws Exception
| |
− | * @BeforeSuite
| |
− | */
| |
− | public static function before_suite($event) {
| |
− | global $CFG, $db, $SESSION, $USER, $THEME;
| |
− | | |
− | // Defined only when the behat CLI command is running, the mahara init setup process will
| |
− | // read this value and switch to $CFG->behat_dataroot and $CFG->behat_dbprefix instead of
| |
− | // the normal site.
| |
− | define('BEHAT_UTIL', 1);
| |
− | | |
− | define('INTERNAL', 1);
| |
− | define('CLI', 1);
| |
− | | |
− | // With BEHAT_TEST we will be using $CFG->behat_* instead of $CFG->dataroot, $CFG->dbprefix and $CFG->wwwroot.
| |
− | require_once(dirname(dirname(dirname(dirname(__DIR__)))) . '/init.php');
| |
− | | |
− | // Now that we are in Mahara env.
| |
− | require_once('upgrade.php');
| |
− | require_once('file.php');
| |
− | require_once(dirname(dirname(dirname(__DIR__))) . '/classes/TestLock.php');
| |
− | require_once(__DIR__ . '/util.php');
| |
− | | |
− | // Initialize and enable the test site if possible
| |
− | $statuscode = BehatTestingUtil::get_test_env_status();
| |
− | switch ($statuscode) {
| |
− | case BEHAT_MAHARA_EXITCODE_OUTOFDATEDB:
| |
− | BehatTestingUtil::drop_site();
| |
− | case BEHAT_MAHARA_EXITCODE_NOTINSTALLED:
| |
− | BehatTestingUtil::install_site();
| |
− | case BEHAT_MAHARA_EXITCODE_NOTENABLED:
| |
− | BehatTestingUtil::start_test_mode();
| |
− | case 0:
| |
− | break;
| |
− | default:
| |
− | throw new Exception($statuscode.'The test site is not ready to test.
| |
− | Please run php ' . dirname(dirname(dirname(dirname(__DIR__)))) . 'testing/frameworks/behat/cli/init.php to initialize the test site');;
| |
− | break;
| |
− | }
| |
− | | |
− | if (!empty($CFG->behat_faildump_path) && !is_writable($CFG->behat_faildump_path)) {
| |
− | throw new Exception('You set $CFG->behat_faildump_path to a non-writable directory');
| |
− | }
| |
− | }
| |
− | | |
− | /**
| |
− | * Clean test database and dataroot and disable the test environment.
| |
− | *
| |
− | * @static
| |
− | * @throws Exception
| |
− | * @AfterSuite
| |
− | */
| |
− | public static function after_suite($event) {
| |
− | global $CFG, $db, $SESSION, $USER, $THEME;
| |
− | | |
− | // Check if the test environment is ready: dataroot, database, server
| |
− | if (!defined('BEHAT_TEST')) {
| |
− | throw new Exception('The test site is not enabled for behat testing');
| |
− | }
| |
− | | |
− | //BehatTestingUtil::drop_site();
| |
− | BehatTestingUtil::stop_test_mode();
| |
− | }
| |
− | | |
− | /**
| |
− | * Resets the test environment.
| |
− | *
| |
− | * @throws Exception If here we are not using the test database it should be because of a coding error
| |
− | * @BeforeScenario
| |
− | */
| |
− | public function before_scenario($event) {
| |
− | global $CFG;
| |
− | | |
− | // Check if the test environment is ready: dataroot, database, server
| |
− | if (!defined('BEHAT_TEST')) {
| |
− | throw new Exception('The test site is not enabled for behat testing');
| |
− | }
| |
− | | |
− | // Check if the browser is running and supports javascript
| |
− | $moreinfo = 'More info in ' . BehatCommand::DOCS_URL . '#Running_tests';
| |
− | $driverexceptionmsg = 'Selenium server is not running, you need to start it to run tests that involve Javascript. ' . $moreinfo;
| |
− | try {
| |
− | $session = $this->getSession();
| |
− | }
| |
− | catch (CurlExec $e) {
| |
− | // Exception thrown by WebDriver, so only @javascript tests will be caugth; in
| |
− | throw new Exception($driverexceptionmsg);
| |
− | }
| |
− | catch (DriverException $e) {
| |
− | throw new Exception($driverexceptionmsg);
| |
− | }
| |
− | catch (UnknownError $e) {
| |
− | // Generic 'I have no idea' Selenium error. Custom exception to provide more feedback about possible solutions.
| |
− | throw new Exception($e);
| |
− | }
| |
− | | |
− | // Register the named selectors for mahara
| |
− | if (self::is_first_scenario()) {
| |
− | BehatSelectors::register_mahara_selectors($session);
| |
− | BehatContextHelper::set_session($session);
| |
− | // Reset the browser
| |
− | $session->restart();
| |
− | // Run all test with medium (1024x768) screen size, to avoid responsive problems.
| |
− | $this->resize_window('medium');
| |
− | }
| |
− | | |
− | // Reset $SESSION.
| |
− | $_SESSION = array();
| |
− | $SESSION = new stdClass();
| |
− | $_SESSION['SESSION'] =& $SESSION;
| |
− | | |
− | BehatTestingUtil::reset_database();
| |
− | BehatTestingUtil::reset_dataroot();
| |
− | | |
− | // Reset the nasty strings list used during the last test.
| |
− | //NastyStrings::reset_used_strings();
| |
− | | |
− | // Set current user is admin
| |
− | | |
− | // Start always in the the homepage.
| |
− | try {
| |
− | // Let's be conservative as we never know when new upstream issues will affect us.
| |
− | $session->visit($this->locate_path('/'));
| |
− | }
| |
− | catch (UnknownError $e) {
| |
− | throw new Exception($e);
| |
− | }
| |
− | | |
− | // Checking that the root path is a mahara test site.
| |
− | if (!self::$initprocessesfinished) {
| |
− | $notestsiteexception = new Exception('The base URL (' . $CFG->wwwroot . ') is not a behat test site, ' .
| |
− | 'ensure you started the built-in web server in the correct directory or your web server is correctly started and set up');
| |
− | $this->find("xpath", "//head/child::title[contains(., '" . BehatTestingUtil::BEHATSITENAME . "')]", $notestsiteexception);
| |
− | | |
− | self::$initprocessesfinished = true;
| |
− | }
| |
− | }
| |
− | | |
− | /**
| |
− | * Wait for JS to complete before beginning interacting with the DOM.
| |
− | *
| |
− | * Executed only when running against a real browser. We wrap it
| |
− | * all in a try & catch to forward the exception to i_look_for_exceptions
| |
− | * so the exception will be at scenario level, which causes a failure, by
| |
− | * default would be at framework level, which will stop the execution of
| |
− | * the run.
| |
− | *
| |
− | * @BeforeStep
| |
− | */
| |
− | | |
− | // public function before_step(BeforeStepScope $scope) {
| |
− | | |
− | // if ($this->running_javascript()) {
| |
− | // try {
| |
− | // $this->wait_for_pending_js();
| |
− | // self::$currentstepexception = null;
| |
− | // }
| |
− | // catch (Exception $e) {
| |
− | // self::$currentstepexception = $e;
| |
− | // }
| |
− | // }
| |
− | // }
| |
− | | |
− | | |
− | /**
| |
− | * @BeforeScenario
| |
− | *
| |
− | * @param BeforeScenarioScope $scope
| |
− | *
| |
− | */
| |
− | public function setUpTestEnvironment($scope)
| |
− | {
| |
− | $this->currentScenario = $scope->getScenario();
| |
− | }
| |
− | | |
− | | |
− | /**
| |
− | * Wait for JS to complete after finishing the step.
| |
− | *
| |
− | * With this we ensure that there are not AJAX calls
| |
− | * still in progress.
| |
− | *
| |
− | * Executed only when running against a real browser. We wrap it
| |
− | * all in a try & catch to forward the exception to i_look_for_exceptions
| |
− | * so the exception will be at scenario level, which causes a failure, by
| |
− | * default would be at framework level, which will stop the execution of
| |
− | * the run.
| |
− | *
| |
− | * Take screenshot if the step failed
| |
− | *
| |
− | * This includes creating an HTML dump of the content if there was a failure.
| |
− | *
| |
− | * @AfterStep
| |
− | */
| |
− | public function after_step(AfterStepScope $scope) {
| |
− | global $CFG;
| |
− | | |
− | if ($this->running_javascript()) {
| |
− | // && in_array($scope->getStep()->getKeywordType(), array('Given', 'When'))) {
| |
− | try {
| |
− | $this->wait_for_pending_js();
| |
− | self::$currentstepexception = null;
| |
− | }
| |
− | catch (UnexpectedAlertOpen $e) {
| |
− | self::$currentstepexception = $e;
| |
− | | |
− | // Accepting the alert so the framework can continue properly running
| |
− | // the following scenarios. Some browsers already closes the alert, so
| |
− | // wrapping in a try & catch.
| |
− | try {
| |
− | $this->getSession()->getDriver()->getWebDriverSession()->accept_alert();
| |
− | }
| |
− | catch (Exception $e) {
| |
− | // Catching the generic one as we never know how drivers reacts here.
| |
− | }
| |
− | }
| |
− | catch (Exception $e) {
| |
− | self::$currentstepexception = $e;
| |
− | }
| |
− | }
| |
− | if (!empty($CFG->behat_faildump_path) &&
| |
− | $scope->getTestResult()->getResultCode() === 99) {
| |
− | $this->take_contentdump($scope);
| |
− | }
| |
− | | |
− | //if test has failed, and is not an api test, get screenshot
| |
− | if(!$scope->getTestResult()->isPassed())
| |
− | {
| |
− | //create filename string
| |
− | | |
− | $featureFolder = preg_replace('/\W/', '', $scope->getFeature()->getTitle());
| |
− |
| |
− | $scenarioName = $this->currentScenario->getTitle();
| |
− | $fileName = preg_replace('/\W/', '', $scenarioName) . '.png';
| |
− | | |
− | //create screenshots directory if it doesn't exist
| |
− | if (!file_exists('/var/lib/maharadata/master_behat/behat/results/html/assets/screenshots/' . $featureFolder)) {
| |
− | mkdir('/var/lib/maharadata/master_behat/behat/results/html/assets/screenshots/' . $featureFolder);
| |
− | }
| |
− | | |
− | //take screenshot and save as the previously defined filename
| |
− | //$this->getDriver()->takeScreenshot('/var/lib/maharadata/master_behat/behat/results/html/assets/screenshots/' . $featureFolder . '/' . $fileName);
| |
− | // For Selenium2 Driver you can use:
| |
− | file_put_contents('/var/lib/maharadata/master_behat/behat/results/html/assets/screenshots/' . $featureFolder . '/' . $fileName, $this->getSession()->getDriver()->getScreenshot());
| |
− | }
| |
− | }
| |
− | | |
− | /**
| |
− | * Getter for self::$faildumpdirname
| |
− | *
| |
− | * @return string
| |
− | */
| |
− | protected function get_run_faildump_dir() {
| |
− | return self::$faildumpdirname;
| |
− | }
| |
− | | |
− | /**
| |
− | * Take screenshot when a step fails.
| |
− | *
| |
− | * @throws Exception
| |
− | * @param AfterStepScope $scope
| |
− | */
| |
− | protected function take_screenshot(AfterStepScope $scope) {
| |
− | // Goutte can't save screenshots.
| |
− | if (!$this->running_javascript()) {
| |
− | return false;
| |
− | }
| |
− | | |
− | list ($dir, $filename) = $this->get_faildump_filename($scope, 'png');
| |
− | $this->saveScreenshot($filename, $dir);
| |
− | }
| |
− | | |
− | /**
| |
− | * Take a dump of the page content when a step fails.
| |
− | *
| |
− | * @throws Exception
| |
− | * @param AfterStepScope $scope
| |
− | */
| |
− | protected function take_contentdump(AfterStepScope $scope) {
| |
− | list ($dir, $filename) = $this->get_faildump_filename($scope, 'html');
| |
− | | |
− | $fh = fopen($dir . DIRECTORY_SEPARATOR . $filename, 'w');
| |
− | fwrite($fh, $this->getSession()->getPage()->getContent());
| |
− | fclose($fh);
| |
− | }
| |
− | | |
− | /**
| |
− | * Determine the full pathname to store a failure-related dump.
| |
− | *
| |
− | * This is used for content such as the DOM, and screenshots.
| |
− | *
| |
− | * @param AfterStepScope $scope
| |
− | * @param String $filetype The file suffix to use. Limited to 4 chars.
| |
− | */
| |
− | protected function get_faildump_filename(AfterStepScope $scope, $filetype) {
| |
− | global $CFG;
| |
− | | |
− | // All the contentdumps should be in the same parent dir.
| |
− | if (!$faildumpdir = self::get_run_faildump_dir()) {
| |
− | $faildumpdir = self::$faildumpdirname = date('Ymd_His');
| |
− | | |
− | $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
| |
− | | |
− | if (!is_dir($dir) && !mkdir($dir, $CFG->directorypermissions, true)) {
| |
− | // It shouldn't, we already checked that the directory is writable.
| |
− | throw new Exception('No directories can be created inside $CFG->behat_faildump_path, check the directory permissions.');
| |
− | }
| |
− | }
| |
− | else {
| |
− | // We will always need to know the full path.
| |
− | $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
| |
− | }
| |
− | | |
− | // The scenario title + the failed step text.
| |
− | // We want a i-am-the-scenario-title_i-am-the-failed-step.$filetype format.
| |
− | $filename = $scope->getStep()->getParent()->getTitle() . '_' . $scope->getStep()->getText();
| |
− | $filename = preg_replace('/([^a-zA-Z0-9\_]+)/', '-', $filename);
| |
− | | |
− | // File name limited to 255 characters. Leaving 4 chars for the file
| |
− | // extension as we allow .png for images and .html for DOM contents.
| |
− | $filename = substr($filename, 0, 250) . '.' . $filetype;
| |
− | | |
− | return array($dir, $filename);
| |
− | }
| |
− | | |
− | /**
| |
− | * Internal step definition to find exceptions, debugging() messages and PHP debug messages.
| |
− | *
| |
− | * Part of BehatHooks class as is part of the testing framework, is auto-executed
| |
− | * after each step so no features will splicitly use it.
| |
− | *
| |
− | * @Given /^I look for exceptions$/
| |
− | * @throw Exception Unknown type, depending on what we caught in the hook or basic \Exception.
| |
− | */
| |
− | public function i_look_for_exceptions() {
| |
− | | |
− | // If the step already failed in a hook throw the exception.
| |
− | if (!is_null(self::$currentstepexception)) {
| |
− | throw self::$currentstepexception;
| |
− | }
| |
− | | |
− | // Wrap in try in case we were interacting with a closed window.
| |
− | try {
| |
− | | |
− | // Exceptions.
| |
− | $exceptionsxpath = "//div[@data-rel='fatalerror']";
| |
− | // Debugging messages.
| |
− | $debuggingxpath = "//div[@data-rel='debugging']";
| |
− | // PHP debug messages.
| |
− | $phperrorxpath = "//div[@data-rel='phpdebugmessage']";
| |
− | // Any other backtrace.
| |
− | $othersxpath = "(//*[contains(., ': call to ')])[1]";
| |
− | | |
− | $xpaths = array($exceptionsxpath, $debuggingxpath, $phperrorxpath, $othersxpath);
| |
− | $joinedxpath = implode(' | ', $xpaths);
| |
− | | |
− | // Joined xpath expression. Most of the time there will be no exceptions, so this pre-check
| |
− | // is faster than to send the 4 xpath queries for each step.
| |
− | if (!$this->getSession()->getDriver()->find($joinedxpath)) {
| |
− | return;
| |
− | }
| |
− | | |
− | // Exceptions.
| |
− | if ($errormsg = $this->getSession()->getPage()->find('xpath', $exceptionsxpath)) {
| |
− | | |
− | // Getting the debugging info and the backtrace.
| |
− | $errorinfoboxes = $this->getSession()->getPage()->findAll('css', 'div.alert-error');
| |
− | // If errorinfoboxes is empty, try find notifytiny (original) class.
| |
− | if (empty($errorinfoboxes)) {
| |
− | $errorinfoboxes = $this->getSession()->getPage()->findAll('css', 'div.notifytiny');
| |
− | }
| |
− | $errorinfo = $this->get_debug_text($errorinfoboxes[0]->getHtml()) . "\n" .
| |
− | $this->get_debug_text($errorinfoboxes[1]->getHtml());
| |
− | | |
− | $msg = "mahara exception: " . $errormsg->getText() . "\n" . $errorinfo;
| |
− | throw new \Exception(html_entity_decode($msg));
| |
− | }
| |
− | | |
− | // Debugging messages.
| |
− | if ($debuggingmessages = $this->getSession()->getPage()->findAll('xpath', $debuggingxpath)) {
| |
− | $msgs = array();
| |
− | foreach ($debuggingmessages as $debuggingmessage) {
| |
− | $msgs[] = $this->get_debug_text($debuggingmessage->getHtml());
| |
− | }
| |
− | $msg = "debugging() message/s found:\n" . implode("\n", $msgs);
| |
− | throw new \Exception(html_entity_decode($msg));
| |
− | }
| |
− | | |
− | // PHP debug messages.
| |
− | if ($phpmessages = $this->getSession()->getPage()->findAll('xpath', $phperrorxpath)) {
| |
− | | |
− | $msgs = array();
| |
− | foreach ($phpmessages as $phpmessage) {
| |
− | $msgs[] = $this->get_debug_text($phpmessage->getHtml());
| |
− | }
| |
− | $msg = "PHP debug message/s found:\n" . implode("\n", $msgs);
| |
− | throw new \Exception(html_entity_decode($msg));
| |
− | }
| |
− | | |
− | // Any other backtrace.
| |
− | // First looking through xpath as it is faster than get and parse the whole page contents,
| |
− | // we get the contents and look for matches once we found something to suspect that there is a backtrace.
| |
− | if ($this->getSession()->getDriver()->find($othersxpath)) {
| |
− | $backtracespattern = '/(line [0-9]* of [^:]*: call to [\->&;:a-zA-Z_\x7f-\xff][\->&;:a-zA-Z0-9_\x7f-\xff]*)/';
| |
− | if (preg_match_all($backtracespattern, $this->getSession()->getPage()->getContent(), $backtraces)) {
| |
− | $msgs = array();
| |
− | foreach ($backtraces[0] as $backtrace) {
| |
− | $msgs[] = $backtrace . '()';
| |
− | }
| |
− | $msg = "Other backtraces found:\n" . implode("\n", $msgs);
| |
− | throw new \Exception(htmlentities($msg));
| |
− | }
| |
− | }
| |
− | | |
− | }
| |
− | catch (NoSuchWindow $e) {
| |
− | // If we were interacting with a popup window it will not exists after closing it.
| |
− | }
| |
− | }
| |
− | | |
− | /**
| |
− | * Converts HTML tags to line breaks to display the info in CLI
| |
− | *
| |
− | * @param string $html
| |
− | * @return string
| |
− | */
| |
− | protected function get_debug_text($html) {
| |
− | | |
− | // Replacing HTML tags for new lines and keeping only the text.
| |
− | $notags = preg_replace('/<+\s*\/*\s*([A-Z][A-Z0-9]*)\b[^>]*\/*\s*>*/i', "\n", $html);
| |
− | return preg_replace("/(\n)+/s", "\n", $notags);
| |
− | }
| |
− | | |
− | /**
| |
− | * Returns whether the first scenario of the suite is running
| |
− | *
| |
− | * @return bool
| |
− | */
| |
− | protected static function is_first_scenario() {
| |
− | return !(self::$initprocessesfinished);
| |
− | }
| |
− | | |
− | /**
| |
− | * Throws an exception after appending an extra info text.
| |
− | *
| |
− | * @throws Exception
| |
− | * @param UnknownError $exception
| |
− | * @return void
| |
− | */
| |
− | protected function throw_unknown_exception(UnknownError $exception) {
| |
− | $text = get_string('unknownexceptioninfo', 'tool_behat');
| |
− | throw new Exception($text . PHP_EOL . $exception->getMessage());
| |
− | }
| |
− | | |
− | }
| |
| | | |
− | 5- Create a folder results under /var/lib/maharadata/master_behat/behat/
| + | Go here for tips: https://wiki.mahara.org/wiki/Testing/Behat_Testing/Error_%26_Solutions |
− | 6- Create a folder html under /var/lib/maharadata/master_behat/behat/results
| |
− | 7- Create a folder screenshots under /var/lib/maharadata/master_behat/behat/results/html/assests
| |
− | 8- Run the test and delete any screenshot before running tests from screenshot folder.
| |
− | Note: Screenshots are now manually deleted, we should find a way to delete the screenshot before running a behat tests
| |