| 1 | '\" |
| 2 | '\" Copyright (c) 1990-1994 The Regents of the University of California |
| 3 | '\" Copyright (c) 1994-1997 Sun Microsystems, Inc. |
| 4 | '\" Copyright (c) 1998-1999 Scriptics Corporation |
| 5 | '\" Copyright (c) 2000 Ajuba Solutions |
| 6 | '\" Contributions from Don Porter, NIST, 2002. (not subject to US copyright) |
| 7 | '\" |
| 8 | '\" See the file "license.terms" for information on usage and redistribution |
| 9 | '\" of this file, and for a DISCLAIMER OF ALL WARRANTIES. |
| 10 | '\" |
| 11 | '\" RCS: @(#) $Id: tcltest.n,v 1.38.2.5 2004/10/27 14:43:13 dkf Exp $ |
| 12 | '\" |
| 13 | '\" The definitions below are for supplemental macros used in Tcl/Tk |
| 14 | '\" manual entries. |
| 15 | '\" |
| 16 | '\" .AP type name in/out ?indent? |
| 17 | '\" Start paragraph describing an argument to a library procedure. |
| 18 | '\" type is type of argument (int, etc.), in/out is either "in", "out", |
| 19 | '\" or "in/out" to describe whether procedure reads or modifies arg, |
| 20 | '\" and indent is equivalent to second arg of .IP (shouldn't ever be |
| 21 | '\" needed; use .AS below instead) |
| 22 | '\" |
| 23 | '\" .AS ?type? ?name? |
| 24 | '\" Give maximum sizes of arguments for setting tab stops. Type and |
| 25 | '\" name are examples of largest possible arguments that will be passed |
| 26 | '\" to .AP later. If args are omitted, default tab stops are used. |
| 27 | '\" |
| 28 | '\" .BS |
| 29 | '\" Start box enclosure. From here until next .BE, everything will be |
| 30 | '\" enclosed in one large box. |
| 31 | '\" |
| 32 | '\" .BE |
| 33 | '\" End of box enclosure. |
| 34 | '\" |
| 35 | '\" .CS |
| 36 | '\" Begin code excerpt. |
| 37 | '\" |
| 38 | '\" .CE |
| 39 | '\" End code excerpt. |
| 40 | '\" |
| 41 | '\" .VS ?version? ?br? |
| 42 | '\" Begin vertical sidebar, for use in marking newly-changed parts |
| 43 | '\" of man pages. The first argument is ignored and used for recording |
| 44 | '\" the version when the .VS was added, so that the sidebars can be |
| 45 | '\" found and removed when they reach a certain age. If another argument |
| 46 | '\" is present, then a line break is forced before starting the sidebar. |
| 47 | '\" |
| 48 | '\" .VE |
| 49 | '\" End of vertical sidebar. |
| 50 | '\" |
| 51 | '\" .DS |
| 52 | '\" Begin an indented unfilled display. |
| 53 | '\" |
| 54 | '\" .DE |
| 55 | '\" End of indented unfilled display. |
| 56 | '\" |
| 57 | '\" .SO |
| 58 | '\" Start of list of standard options for a Tk widget. The |
| 59 | '\" options follow on successive lines, in four columns separated |
| 60 | '\" by tabs. |
| 61 | '\" |
| 62 | '\" .SE |
| 63 | '\" End of list of standard options for a Tk widget. |
| 64 | '\" |
| 65 | '\" .OP cmdName dbName dbClass |
| 66 | '\" Start of description of a specific option. cmdName gives the |
| 67 | '\" option's name as specified in the class command, dbName gives |
| 68 | '\" the option's name in the option database, and dbClass gives |
| 69 | '\" the option's class in the option database. |
| 70 | '\" |
| 71 | '\" .UL arg1 arg2 |
| 72 | '\" Print arg1 underlined, then print arg2 normally. |
| 73 | '\" |
| 74 | '\" RCS: @(#) $Id: man.macros,v 1.4 2000/08/25 06:18:32 ericm Exp $ |
| 75 | '\" |
| 76 | '\" # Set up traps and other miscellaneous stuff for Tcl/Tk man pages. |
| 77 | .if t .wh -1.3i ^B |
| 78 | .nr ^l \n(.l |
| 79 | .ad b |
| 80 | '\" # Start an argument description |
| 81 | .de AP |
| 82 | .ie !"\\$4"" .TP \\$4 |
| 83 | .el \{\ |
| 84 | . ie !"\\$2"" .TP \\n()Cu |
| 85 | . el .TP 15 |
| 86 | .\} |
| 87 | .ta \\n()Au \\n()Bu |
| 88 | .ie !"\\$3"" \{\ |
| 89 | \&\\$1 \\fI\\$2\\fP (\\$3) |
| 90 | .\".b |
| 91 | .\} |
| 92 | .el \{\ |
| 93 | .br |
| 94 | .ie !"\\$2"" \{\ |
| 95 | \&\\$1 \\fI\\$2\\fP |
| 96 | .\} |
| 97 | .el \{\ |
| 98 | \&\\fI\\$1\\fP |
| 99 | .\} |
| 100 | .\} |
| 101 | .. |
| 102 | '\" # define tabbing values for .AP |
| 103 | .de AS |
| 104 | .nr )A 10n |
| 105 | .if !"\\$1"" .nr )A \\w'\\$1'u+3n |
| 106 | .nr )B \\n()Au+15n |
| 107 | .\" |
| 108 | .if !"\\$2"" .nr )B \\w'\\$2'u+\\n()Au+3n |
| 109 | .nr )C \\n()Bu+\\w'(in/out)'u+2n |
| 110 | .. |
| 111 | .AS Tcl_Interp Tcl_CreateInterp in/out |
| 112 | '\" # BS - start boxed text |
| 113 | '\" # ^y = starting y location |
| 114 | '\" # ^b = 1 |
| 115 | .de BS |
| 116 | .br |
| 117 | .mk ^y |
| 118 | .nr ^b 1u |
| 119 | .if n .nf |
| 120 | .if n .ti 0 |
| 121 | .if n \l'\\n(.lu\(ul' |
| 122 | .if n .fi |
| 123 | .. |
| 124 | '\" # BE - end boxed text (draw box now) |
| 125 | .de BE |
| 126 | .nf |
| 127 | .ti 0 |
| 128 | .mk ^t |
| 129 | .ie n \l'\\n(^lu\(ul' |
| 130 | .el \{\ |
| 131 | .\" Draw four-sided box normally, but don't draw top of |
| 132 | .\" box if the box started on an earlier page. |
| 133 | .ie !\\n(^b-1 \{\ |
| 134 | \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul' |
| 135 | .\} |
| 136 | .el \}\ |
| 137 | \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul' |
| 138 | .\} |
| 139 | .\} |
| 140 | .fi |
| 141 | .br |
| 142 | .nr ^b 0 |
| 143 | .. |
| 144 | '\" # VS - start vertical sidebar |
| 145 | '\" # ^Y = starting y location |
| 146 | '\" # ^v = 1 (for troff; for nroff this doesn't matter) |
| 147 | .de VS |
| 148 | .if !"\\$2"" .br |
| 149 | .mk ^Y |
| 150 | .ie n 'mc \s12\(br\s0 |
| 151 | .el .nr ^v 1u |
| 152 | .. |
| 153 | '\" # VE - end of vertical sidebar |
| 154 | .de VE |
| 155 | .ie n 'mc |
| 156 | .el \{\ |
| 157 | .ev 2 |
| 158 | .nf |
| 159 | .ti 0 |
| 160 | .mk ^t |
| 161 | \h'|\\n(^lu+3n'\L'|\\n(^Yu-1v\(bv'\v'\\n(^tu+1v-\\n(^Yu'\h'-|\\n(^lu+3n' |
| 162 | .sp -1 |
| 163 | .fi |
| 164 | .ev |
| 165 | .\} |
| 166 | .nr ^v 0 |
| 167 | .. |
| 168 | '\" # Special macro to handle page bottom: finish off current |
| 169 | '\" # box/sidebar if in box/sidebar mode, then invoked standard |
| 170 | '\" # page bottom macro. |
| 171 | .de ^B |
| 172 | .ev 2 |
| 173 | 'ti 0 |
| 174 | 'nf |
| 175 | .mk ^t |
| 176 | .if \\n(^b \{\ |
| 177 | .\" Draw three-sided box if this is the box's first page, |
| 178 | .\" draw two sides but no top otherwise. |
| 179 | .ie !\\n(^b-1 \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c |
| 180 | .el \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c |
| 181 | .\} |
| 182 | .if \\n(^v \{\ |
| 183 | .nr ^x \\n(^tu+1v-\\n(^Yu |
| 184 | \kx\h'-\\nxu'\h'|\\n(^lu+3n'\ky\L'-\\n(^xu'\v'\\n(^xu'\h'|0u'\c |
| 185 | .\} |
| 186 | .bp |
| 187 | 'fi |
| 188 | .ev |
| 189 | .if \\n(^b \{\ |
| 190 | .mk ^y |
| 191 | .nr ^b 2 |
| 192 | .\} |
| 193 | .if \\n(^v \{\ |
| 194 | .mk ^Y |
| 195 | .\} |
| 196 | .. |
| 197 | '\" # DS - begin display |
| 198 | .de DS |
| 199 | .RS |
| 200 | .nf |
| 201 | .sp |
| 202 | .. |
| 203 | '\" # DE - end display |
| 204 | .de DE |
| 205 | .fi |
| 206 | .RE |
| 207 | .sp |
| 208 | .. |
| 209 | '\" # SO - start of list of standard options |
| 210 | .de SO |
| 211 | .SH "STANDARD OPTIONS" |
| 212 | .LP |
| 213 | .nf |
| 214 | .ta 5.5c 11c |
| 215 | .ft B |
| 216 | .. |
| 217 | '\" # SE - end of list of standard options |
| 218 | .de SE |
| 219 | .fi |
| 220 | .ft R |
| 221 | .LP |
| 222 | See the \\fBoptions\\fR manual entry for details on the standard options. |
| 223 | .. |
| 224 | '\" # OP - start of full description for a single option |
| 225 | .de OP |
| 226 | .LP |
| 227 | .nf |
| 228 | .ta 4c |
| 229 | Command-Line Name: \\fB\\$1\\fR |
| 230 | Database Name: \\fB\\$2\\fR |
| 231 | Database Class: \\fB\\$3\\fR |
| 232 | .fi |
| 233 | .IP |
| 234 | .. |
| 235 | '\" # CS - begin code excerpt |
| 236 | .de CS |
| 237 | .RS |
| 238 | .nf |
| 239 | .ta .25i .5i .75i 1i |
| 240 | .. |
| 241 | '\" # CE - end code excerpt |
| 242 | .de CE |
| 243 | .fi |
| 244 | .RE |
| 245 | .. |
| 246 | .de UL |
| 247 | \\$1\l'|0\(ul'\\$2 |
| 248 | .. |
| 249 | .TH "tcltest" n 2.2 tcltest "Tcl Bundled Packages" |
| 250 | .BS |
| 251 | '\" Note: do not modify the .SH NAME line immediately below! |
| 252 | .SH NAME |
| 253 | tcltest \- Test harness support code and utilities |
| 254 | .SH SYNOPSIS |
| 255 | .nf |
| 256 | \fBpackage require tcltest ?2.2.5?\fR |
| 257 | .sp |
| 258 | \fBtcltest::test \fIname description ?option value ...?\fR |
| 259 | \fBtcltest::test \fIname description ?constraints? body result\fR |
| 260 | .sp |
| 261 | \fBtcltest::loadTestedCommands\fR |
| 262 | \fBtcltest::makeDirectory \fIname ?directory?\fR |
| 263 | \fBtcltest::removeDirectory \fIname ?directory?\fR |
| 264 | \fBtcltest::makeFile \fIcontents name ?directory?\fR |
| 265 | \fBtcltest::removeFile \fIname ?directory?\fR |
| 266 | \fBtcltest::viewFile \fIname ?directory?\fR |
| 267 | \fBtcltest::cleanupTests \fI?runningMultipleTests?\fR |
| 268 | \fBtcltest::runAllTests\fR |
| 269 | .sp |
| 270 | \fBtcltest::configure\fR |
| 271 | \fBtcltest::configure \fIoption\fR |
| 272 | \fBtcltest::configure \fIoption value ?option value ...?\fR |
| 273 | \fBtcltest::customMatch \fImode command\fR |
| 274 | \fBtcltest::testConstraint \fIconstraint ?value?\fR |
| 275 | \fBtcltest::outputChannel \fI?channelID?\fR |
| 276 | \fBtcltest::errorChannel \fI?channelID?\fR |
| 277 | \fBtcltest::interpreter \fI?interp?\fR |
| 278 | .sp |
| 279 | \fBtcltest::debug \fI?level?\fR |
| 280 | \fBtcltest::errorFile \fI?filename?\fR |
| 281 | \fBtcltest::limitConstraints \fI?boolean?\fR |
| 282 | \fBtcltest::loadFile \fI?filename?\fR |
| 283 | \fBtcltest::loadScript \fI?script?\fR |
| 284 | \fBtcltest::match \fI?patternList?\fR |
| 285 | \fBtcltest::matchDirectories \fI?patternList?\fR |
| 286 | \fBtcltest::matchFiles \fI?patternList?\fR |
| 287 | \fBtcltest::outputFile \fI?filename?\fR |
| 288 | \fBtcltest::preserveCore \fI?level?\fR |
| 289 | \fBtcltest::singleProcess \fI?boolean?\fR |
| 290 | \fBtcltest::skip \fI?patternList?\fR |
| 291 | \fBtcltest::skipDirectories \fI?patternList?\fR |
| 292 | \fBtcltest::skipFiles \fI?patternList?\fR |
| 293 | \fBtcltest::temporaryDirectory \fI?directory?\fR |
| 294 | \fBtcltest::testsDirectory \fI?directory?\fR |
| 295 | \fBtcltest::verbose \fI?level?\fR |
| 296 | .sp |
| 297 | \fBtcltest::test \fIname description optionList\fR |
| 298 | \fBtcltest::bytestring \fIstring\fR |
| 299 | \fBtcltest::normalizeMsg \fImsg\fR |
| 300 | \fBtcltest::normalizePath \fIpathVar\fR |
| 301 | \fBtcltest::workingDirectory \fI?dir?\fR |
| 302 | .fi |
| 303 | .BE |
| 304 | .SH DESCRIPTION |
| 305 | .PP |
| 306 | The \fBtcltest\fR package provides several utility commands useful |
| 307 | in the construction of test suites for code instrumented to be |
| 308 | run by evaluation of Tcl commands. Notably the built-in commands |
| 309 | of the Tcl library itself are tested by a test suite using the |
| 310 | tcltest package. |
| 311 | .PP |
| 312 | All the commands provided by the \fBtcltest\fR package are defined |
| 313 | in and exported from the \fB::tcltest\fR namespace, as indicated in |
| 314 | the \fBSYNOPSIS\fR above. In the following sections, all commands |
| 315 | will be described by their simple names, in the interest of brevity. |
| 316 | .PP |
| 317 | The central command of \fBtcltest\fR is [\fBtest\fR] that defines |
| 318 | and runs a test. Testing with [\fBtest\fR] involves evaluation |
| 319 | of a Tcl script and comparing the result to an expected result, as |
| 320 | configured and controlled by a number of options. Several other |
| 321 | commands provided by \fBtcltest\fR govern the configuration of |
| 322 | [\fBtest\fR] and the collection of many [\fBtest\fR] commands into |
| 323 | test suites. |
| 324 | .PP |
| 325 | See \fBCREATING TEST SUITES WITH TCLTEST\fR below for an extended example |
| 326 | of how to use the commands of \fBtcltest\fR to produce test suites |
| 327 | for your Tcl-enabled code. |
| 328 | .SH COMMANDS |
| 329 | .TP |
| 330 | \fBtest\fR \fIname description ?option value ...?\fR |
| 331 | Defines and possibly runs a test with the name \fIname\fR and |
| 332 | description \fIdescription\fR. The name and description of a test |
| 333 | are used in messages reported by [\fBtest\fR] during the |
| 334 | test, as configured by the options of \fBtcltest\fR. The |
| 335 | remaining \fIoption value\fR arguments to [\fBtest\fR] |
| 336 | define the test, including the scripts to run, the conditions |
| 337 | under which to run them, the expected result, and the means |
| 338 | by which the expected and actual results should be compared. |
| 339 | See \fBTESTS\fR below for a complete description of the valid |
| 340 | options and how they define a test. The [\fBtest\fR] command |
| 341 | returns an empty string. |
| 342 | .TP |
| 343 | \fBtest\fR \fIname description ?constraints? body result\fR |
| 344 | This form of [\fBtest\fR] is provided to support test suites written |
| 345 | for version 1 of the \fBtcltest\fR package, and also a simpler |
| 346 | interface for a common usage. It is the same as |
| 347 | [\fBtest\fR \fIname description\fB -constraints \fIconstraints\fB -body |
| 348 | \fIbody\fB -result \fIresult\fR]. All other options to [\fBtest\fR] |
| 349 | take their default values. When \fIconstraints\fR is omitted, this |
| 350 | form of [\fBtest\fR] can be distinguished from the first because |
| 351 | all \fIoption\fRs begin with ``-''. |
| 352 | .TP |
| 353 | \fBloadTestedCommands\fR |
| 354 | Evaluates in the caller's context the script specified by |
| 355 | [\fBconfigure -load\fR] or [\fBconfigure -loadfile\fR]. |
| 356 | Returns the result of that script evaluation, including any error |
| 357 | raised by the script. Use this command and the related |
| 358 | configuration options to provide the commands to be tested to |
| 359 | the interpreter running the test suite. |
| 360 | .TP |
| 361 | \fBmakeFile\fR \fIcontents name ?directory?\fR |
| 362 | Creates a file named \fIname\fR relative to |
| 363 | directory \fIdirectory\fR and write \fIcontents\fR |
| 364 | to that file using the encoding [\fBencoding system\fR]. |
| 365 | If \fIcontents\fR does not end with a newline, a newline |
| 366 | will be appended so that the file named \fIname\fR |
| 367 | does end with a newline. Because the system encoding is used, |
| 368 | this command is only suitable for making text files. |
| 369 | The file will be removed by the next evaluation |
| 370 | of [\fBcleanupTests\fR], unless it is removed by |
| 371 | [\fBremoveFile\fR] first. The default value of |
| 372 | \fIdirectory\fR is the directory [\fBconfigure -tmpdir\fR]. |
| 373 | Returns the full path of the file created. Use this command |
| 374 | to create any text file required by a test with contents as needed. |
| 375 | .TP |
| 376 | \fBremoveFile\fR \fIname ?directory?\fR |
| 377 | Forces the file referenced by \fIname\fR to be removed. This file name |
| 378 | should be relative to \fIdirectory\fR. The default value of |
| 379 | \fIdirectory\fR is the directory [\fBconfigure -tmpdir\fR]. |
| 380 | Returns an empty string. Use this command to delete files |
| 381 | created by [\fBmakeFile\fR]. |
| 382 | .TP |
| 383 | \fBmakeDirectory\fR \fIname ?directory?\fR |
| 384 | Creates a directory named \fIname\fR relative to directory \fIdirectory\fR. |
| 385 | The directory will be removed by the next evaluation of [\fBcleanupTests\fR], |
| 386 | unless it is removed by [\fBremoveDirectory\fR] first. |
| 387 | The default value of \fIdirectory\fR is the directory |
| 388 | [\fBconfigure -tmpdir\fR]. |
| 389 | Returns the full path of the directory created. Use this command |
| 390 | to create any directories that are required to exist by a test. |
| 391 | .TP |
| 392 | \fBremoveDirectory\fR \fIname ?directory?\fR |
| 393 | Forces the directory referenced by \fIname\fR to be removed. This |
| 394 | directory should be relative to \fIdirectory\fR. |
| 395 | The default value of \fIdirectory\fR is the directory |
| 396 | [\fBconfigure -tmpdir\fR]. |
| 397 | Returns an empty string. Use this command to delete any directories |
| 398 | created by [\fBmakeDirectory\fR]. |
| 399 | .TP |
| 400 | \fBviewFile\fR \fIfile ?directory?\fR |
| 401 | Returns the contents of \fIfile\fR, except for any |
| 402 | final newline, just as [\fBread -nonewline\fR] would return. |
| 403 | This file name should be relative to \fIdirectory\fR. |
| 404 | The default value of \fIdirectory\fR is the directory |
| 405 | [\fBconfigure -tmpdir\fR]. Use this command |
| 406 | as a convenient way to turn the contents of a file generated |
| 407 | by a test into the result of that test for matching against |
| 408 | an expected result. The contents of the file are read using |
| 409 | the system encoding, so its usefulness is limited to text |
| 410 | files. |
| 411 | .TP |
| 412 | \fBcleanupTests\fR |
| 413 | Intended to clean up and summarize after several tests have been |
| 414 | run. Typically called once per test file, at the end of the file |
| 415 | after all tests have been completed. For best effectiveness, be |
| 416 | sure that the [\fBcleanupTests\fR] is evaluated even if an error |
| 417 | occurs earlier in the test file evaluation. |
| 418 | .sp |
| 419 | Prints statistics about the tests run and removes files that were |
| 420 | created by [\fBmakeDirectory\fR] and [\fBmakeFile\fR] since the |
| 421 | last [\fBcleanupTests\fR]. Names of files and directories |
| 422 | in the directory [\fBconfigure -tmpdir\fR] created since |
| 423 | the last [\fBcleanupTests\fR], but not created by |
| 424 | [\fBmakeFile\fR] or [\fBmakeDirectory\fR] are printed |
| 425 | to [\fBoutputChannel\fR]. This command also restores the original |
| 426 | shell environment, as described by the ::env |
| 427 | array. Returns an empty string. |
| 428 | .TP |
| 429 | \fBrunAllTests\fR |
| 430 | This is a master command meant to run an entire suite of tests, |
| 431 | spanning multiple files and/or directories, as governed by |
| 432 | the configurable options of \fBtcltest\fR. See \fBRUNNING ALL TESTS\fR |
| 433 | below for a complete description of the many variations possible |
| 434 | with [\fBrunAllTests\fR]. |
| 435 | .SH "CONFIGURATION COMMANDS" |
| 436 | .TP |
| 437 | \fBconfigure\fR |
| 438 | Returns the list of configurable options supported by \fBtcltest\fR. |
| 439 | See \fBCONFIGURABLE OPTIONS\fR below for the full list of options, |
| 440 | their valid values, and their effect on \fBtcltest\fR operations. |
| 441 | .TP |
| 442 | \fBconfigure \fIoption\fR |
| 443 | Returns the current value of the supported configurable option \fIoption\fR. |
| 444 | Raises an error if \fIoption\fR is not a supported configurable option. |
| 445 | .TP |
| 446 | \fBconfigure \fIoption value ?option value ...?\fR |
| 447 | Sets the value of each configurable option \fIoption\fR to the |
| 448 | corresponding value \fIvalue\fR, in order. Raises an error if |
| 449 | an \fIoption\fR is not a supported configurable option, or if |
| 450 | \fIvalue\fR is not a valid value for the corresponding \fIoption\fR, |
| 451 | or if a \fIvalue\fR is not provided. When an error is raised, the |
| 452 | operation of [\fBconfigure\fR] is halted, and subsequent \fIoption value\fR |
| 453 | arguments are not processed. |
| 454 | .sp |
| 455 | If the environment variable \fB::env(TCLTEST_OPTIONS)\fR exists when |
| 456 | the \fBtcltest\fR package is loaded (by [\fBpackage require tcltest\fR]) |
| 457 | then its value is taken as a list of arguments to pass to [\fBconfigure\fR]. |
| 458 | This allows the default values of the configuration options to be |
| 459 | set by the environment. |
| 460 | .TP |
| 461 | \fBcustomMatch \fImode script\fR |
| 462 | Registers \fImode\fR as a new legal value of the \fB-match\fR option |
| 463 | to [\fBtest\fR]. When the \fB-match \fImode\fR option is |
| 464 | passed to [\fBtest\fR], the script \fIscript\fR will be evaluated |
| 465 | to compare the actual result of evaluating the body of the test |
| 466 | to the expected result. |
| 467 | To perform the match, the \fIscript\fR is completed with two additional |
| 468 | words, the expected result, and the actual result, and the completed script |
| 469 | is evaluated in the global namespace. |
| 470 | The completed script is expected to return a boolean value indicating |
| 471 | whether or not the results match. The built-in matching modes of |
| 472 | [\fBtest\fR] are \fBexact\fR, \fBglob\fR, and \fBregexp\fR. |
| 473 | .TP |
| 474 | \fBtestConstraint \fIconstraint ?boolean?\fR |
| 475 | Sets or returns the boolean value associated with the named \fIconstraint\fR. |
| 476 | See \fBTEST CONSTRAINTS\fR below for more information. |
| 477 | .TP |
| 478 | \fBinterpreter\fR \fI?executableName?\fR |
| 479 | Sets or returns the name of the executable to be [\fBexec\fR]ed by |
| 480 | [\fBrunAllTests\fR] to run each test file when |
| 481 | [\fBconfigure -singleproc\fR] is false. |
| 482 | The default value for [\fBinterpreter\fR] is the name of the |
| 483 | currently running program as returned by [\fBinfo nameofexecutable\fR]. |
| 484 | .TP |
| 485 | \fBoutputChannel\fR \fI?channelID?\fR |
| 486 | Sets or returns the output channel ID. This defaults to stdout. |
| 487 | Any test that prints test related output should send |
| 488 | that output to [\fBoutputChannel\fR] rather than letting |
| 489 | that output default to stdout. |
| 490 | .TP |
| 491 | \fBerrorChannel\fR \fI?channelID?\fR |
| 492 | Sets or returns the error channel ID. This defaults to stderr. |
| 493 | Any test that prints error messages should send |
| 494 | that output to [\fBerrorChannel\fR] rather than printing |
| 495 | directly to stderr. |
| 496 | .SH "SHORTCUT COMMANDS" |
| 497 | .TP |
| 498 | \fBdebug \fI?level?\fR |
| 499 | Same as [\fBconfigure -debug \fI?level?\fR]. |
| 500 | .TP |
| 501 | \fBerrorFile \fI?filename?\fR |
| 502 | Same as [\fBconfigure -errfile \fI?filename?\fR]. |
| 503 | .TP |
| 504 | \fBlimitConstraints \fI?boolean?\fR |
| 505 | Same as [\fBconfigure -limitconstraints \fI?boolean?\fR]. |
| 506 | .TP |
| 507 | \fBloadFile \fI?filename?\fR |
| 508 | Same as [\fBconfigure -loadfile \fI?filename?\fR]. |
| 509 | .TP |
| 510 | \fBloadScript \fI?script?\fR |
| 511 | Same as [\fBconfigure -load \fI?script?\fR]. |
| 512 | .TP |
| 513 | \fBmatch \fI?patternList?\fR |
| 514 | Same as [\fBconfigure -match \fI?patternList?\fR]. |
| 515 | .TP |
| 516 | \fBmatchDirectories \fI?patternList?\fR |
| 517 | Same as [\fBconfigure -relateddir \fI?patternList?\fR]. |
| 518 | .TP |
| 519 | \fBmatchFiles \fI?patternList?\fR |
| 520 | Same as [\fBconfigure -file \fI?patternList?\fR]. |
| 521 | .TP |
| 522 | \fBoutputFile \fI?filename?\fR |
| 523 | Same as [\fBconfigure -outfile \fI?filename?\fR]. |
| 524 | .TP |
| 525 | \fBpreserveCore \fI?level?\fR |
| 526 | Same as [\fBconfigure -preservecore \fI?level?\fR]. |
| 527 | .TP |
| 528 | \fBsingleProcess \fI?boolean?\fR |
| 529 | Same as [\fBconfigure -singleproc \fI?boolean?\fR]. |
| 530 | .TP |
| 531 | \fBskip \fI?patternList?\fR |
| 532 | Same as [\fBconfigure -skip \fI?patternList?\fR]. |
| 533 | .TP |
| 534 | \fBskipDirectories \fI?patternList?\fR |
| 535 | Same as [\fBconfigure -asidefromdir \fI?patternList?\fR]. |
| 536 | .TP |
| 537 | \fBskipFiles \fI?patternList?\fR |
| 538 | Same as [\fBconfigure -notfile \fI?patternList?\fR]. |
| 539 | .TP |
| 540 | \fBtemporaryDirectory \fI?directory?\fR |
| 541 | Same as [\fBconfigure -tmpdir \fI?directory?\fR]. |
| 542 | .TP |
| 543 | \fBtestsDirectory \fI?directory?\fR |
| 544 | Same as [\fBconfigure -testdir \fI?directory?\fR]. |
| 545 | .TP |
| 546 | \fBverbose \fI?level?\fR |
| 547 | Same as [\fBconfigure -verbose \fI?level?\fR]. |
| 548 | .SH "OTHER COMMANDS" |
| 549 | .PP |
| 550 | The remaining commands provided by \fBtcltest\fR have better |
| 551 | alternatives provided by \fBtcltest\fR or \fBTcl\fR itself. They |
| 552 | are retained to support existing test suites, but should be avoided |
| 553 | in new code. |
| 554 | .TP |
| 555 | \fBtest\fR \fIname description optionList\fR |
| 556 | This form of [\fBtest\fR] was provided to enable passing many |
| 557 | options spanning several lines to [\fBtest\fR] as a single |
| 558 | argument quoted by braces, rather than needing to backslash quote |
| 559 | the newlines between arguments to [\fBtest\fR]. The \fIoptionList\fR |
| 560 | argument is expected to be a list with an even number of elements |
| 561 | representing \fIoption\fR and \fIvalue\fR arguments to pass |
| 562 | to [\fBtest\fR]. However, these values are not passed directly, as |
| 563 | in the alternate forms of [\fBswitch\fR]. Instead, this form makes |
| 564 | an unfortunate attempt to overthrow Tcl's substitution rules by |
| 565 | performing substitutions on some of the list elements as an attempt to |
| 566 | implement a ``do what I mean'' interpretation of a brace-enclosed |
| 567 | ``block''. The result is nearly impossible to document clearly, and |
| 568 | for that reason this form is not recommended. See the examples in |
| 569 | \fBCREATING TEST SUITES WITH TCLTEST\fR below to see that this |
| 570 | form is really not necessary to avoid backslash-quoted newlines. |
| 571 | If you insist on using this form, examine |
| 572 | the source code of \fBtcltest\fR if you want to know the substitution |
| 573 | details, or just enclose the third through last argument |
| 574 | to [\fBtest\fR] in braces and hope for the best. |
| 575 | .TP |
| 576 | \fBworkingDirectory\fR \fI?directoryName?\fR |
| 577 | Sets or returns the current working directory when the test suite is |
| 578 | running. The default value for workingDirectory is the directory in |
| 579 | which the test suite was launched. The Tcl commands [\fBcd\fR] and |
| 580 | [\fBpwd\fR] are sufficient replacements. |
| 581 | .TP |
| 582 | \fBnormalizeMsg\fR \fImsg\fR |
| 583 | Returns the result of removing the ``extra'' newlines from \fImsg\fR, |
| 584 | where ``extra'' is rather imprecise. Tcl offers plenty of string |
| 585 | processing commands to modify strings as you wish, and |
| 586 | [\fBcustomMatch\fR] allows flexible matching of actual and expected |
| 587 | results. |
| 588 | .TP |
| 589 | \fBnormalizePath\fR \fIpathVar\fR |
| 590 | Resolves symlinks in a path, thus creating a path without internal |
| 591 | redirection. It is assumed that \fIpathVar\fR is absolute. |
| 592 | \fIpathVar\fR is modified in place. The Tcl command [\fBfile normalize\fR] |
| 593 | is a sufficient replacement. |
| 594 | .TP |
| 595 | \fBbytestring\fR \fIstring\fR |
| 596 | Construct a string that consists of the requested sequence of bytes, |
| 597 | as opposed to a string of properly formed UTF-8 characters using the |
| 598 | value supplied in \fIstring\fR. This allows the tester to create |
| 599 | denormalized or improperly formed strings to pass to C procedures that |
| 600 | are supposed to accept strings with embedded NULL types and confirm |
| 601 | that a string result has a certain pattern of bytes. This is |
| 602 | exactly equivalent to the Tcl command [\fBencoding convertfrom identity\fR]. |
| 603 | .SH TESTS |
| 604 | .PP |
| 605 | The [\fBtest\fR] command is the heart of the \fBtcltest\fR package. |
| 606 | Its essential function is to evaluate a Tcl script and compare |
| 607 | the result with an expected result. The options of [\fBtest\fR] |
| 608 | define the test script, the environment in which to evaluate it, |
| 609 | the expected result, and how the compare the actual result to |
| 610 | the expected result. Some configuration options of \fBtcltest\fR |
| 611 | also influence how [\fBtest\fR] operates. |
| 612 | .PP |
| 613 | The valid options for [\fBtest\fR] are summarized: |
| 614 | .CS |
| 615 | .ta 0.8i |
| 616 | \fBtest\fR \fIname\fR \fIdescription\fR |
| 617 | ?-constraints \fIkeywordList|expression\fR? |
| 618 | ?-setup \fIsetupScript\fR? |
| 619 | ?-body \fItestScript\fR? |
| 620 | ?-cleanup \fIcleanupScript\fR? |
| 621 | ?-result \fIexpectedAnswer\fR? |
| 622 | ?-output \fIexpectedOutput\fR? |
| 623 | ?-errorOutput \fIexpectedError\fR? |
| 624 | ?-returnCodes \fIcodeList\fR? |
| 625 | ?-match \fImode\fR? |
| 626 | .CE |
| 627 | The \fIname\fR may be any string. It is conventional to choose |
| 628 | a \fIname\fR according to the pattern: |
| 629 | .CS |
| 630 | \fItarget\fR-\fImajorNum\fR.\fIminorNum\fR |
| 631 | .CE |
| 632 | For white-box (regression) tests, the target should be the name of the |
| 633 | C function or Tcl procedure being tested. For black-box tests, the |
| 634 | target should be the name of the feature being tested. Some conventions |
| 635 | call for the names of black-box tests to have the suffix \fB_bb\fR. |
| 636 | Related tests should share a major number. As a test suite evolves, |
| 637 | it is best to have the same test name continue to correspond to the |
| 638 | same test, so that it remains meaningful to say things like ``Test |
| 639 | foo-1.3 passed in all releases up to 3.4, but began failing in |
| 640 | release 3.5.'' |
| 641 | .PP |
| 642 | During evaluation of [\fBtest\fR], the \fIname\fR will be compared |
| 643 | to the lists of string matching patterns returned by |
| 644 | [\fBconfigure -match\fR], and [\fBconfigure -skip\fR]. The test |
| 645 | will be run only if \fIname\fR matches any of the patterns from |
| 646 | [\fBconfigure -match\fR] and matches none of the patterns |
| 647 | from [\fBconfigure -skip\fR]. |
| 648 | .PP |
| 649 | The \fIdescription\fR should be a short textual description of the |
| 650 | test. The \fIdescription\fR is included in output produced by the |
| 651 | test, typically test failure messages. Good \fIdescription\fR values |
| 652 | should briefly explain the purpose of the test to users of a test suite. |
| 653 | The name of a Tcl or C function being tested should be included in the |
| 654 | description for regression tests. If the test case exists to reproduce |
| 655 | a bug, include the bug ID in the description. |
| 656 | .PP |
| 657 | Valid attributes and associated values are: |
| 658 | .TP |
| 659 | \fB-constraints \fIkeywordList|expression\fR |
| 660 | The optional \fB-constraints\fR attribute can be list of one or more |
| 661 | keywords or an expression. If the \fB-constraints\fR value is a list of |
| 662 | keywords, each of these keywords should be the name of a constraint |
| 663 | defined by a call to [\fBtestConstraint\fR]. If any of the listed |
| 664 | constraints is false or does not exist, the test is skipped. If the |
| 665 | \fB-constraints\fR value is an expression, that expression |
| 666 | is evaluated. If the expression evaluates to true, then the test is run. |
| 667 | Note that the expression form of \fB-constraints\fR may interfere with the |
| 668 | operation of [\fBconfigure -constraints\fR] and |
| 669 | [\fBconfigure -limitconstraints\fR], and is not recommended. |
| 670 | Appropriate constraints should be added to any tests that should |
| 671 | not always be run. That is, conditional evaluation of a test |
| 672 | should be accomplished by the \fB-constraints\fR option, not by |
| 673 | conditional evaluation of [\fBtest\fR]. In that way, the same |
| 674 | number of tests are always reported by the test suite, though |
| 675 | the number skipped may change based on the testing environment. |
| 676 | The default value is an empty list. |
| 677 | See \fBTEST CONSTRAINTS\fR below for a list of built-in constraints |
| 678 | and information on how to add your own constraints. |
| 679 | .TP |
| 680 | \fB-setup \fIscript\fR |
| 681 | The optional \fB-setup\fR attribute indicates a \fIscript\fR that will be run |
| 682 | before the script indicated by the \fB-body\fR attribute. If evaluation |
| 683 | of \fIscript\fR raises an error, the test will fail. The default value |
| 684 | is an empty script. |
| 685 | .TP |
| 686 | \fB-body \fIscript\fR |
| 687 | The \fB-body\fR attribute indicates the \fIscript\fR to run to carry out the |
| 688 | test. It must return a result that can be checked for correctness. |
| 689 | If evaluation of \fIscript\fR raises an error, the test will fail. |
| 690 | The default value is an empty script. |
| 691 | .TP |
| 692 | \fB-cleanup \fIscript\fR |
| 693 | The optional \fB-cleanup\fR attribute indicates a \fIscript\fR that will be |
| 694 | run after the script indicated by the \fB-body\fR attribute. |
| 695 | If evaluation of \fIscript\fR raises an error, the test will fail. |
| 696 | The default value is an empty script. |
| 697 | .TP |
| 698 | \fB-match \fImode\fR |
| 699 | The \fB-match\fR attribute determines how expected answers supplied by |
| 700 | \fB-result\fR, \fB-output\fR, and \fB-errorOutput\fR are compared. Valid |
| 701 | values for \fImode\fR are \fBregexp\fR, \fBglob\fR, \fBexact\fR, and |
| 702 | any value registered by a prior call to [\fBcustomMatch\fR]. The default |
| 703 | value is \fBexact\fR. |
| 704 | .TP |
| 705 | \fB-result \fIexpectedValue\fR |
| 706 | The \fB-result\fR attribute supplies the \fIexpectedValue\fR against which |
| 707 | the return value from script will be compared. The default value is |
| 708 | an empty string. |
| 709 | .TP |
| 710 | \fB-output \fIexpectedValue\fR |
| 711 | The \fB-output\fR attribute supplies the \fIexpectedValue\fR against which |
| 712 | any output sent to \fBstdout\fR or [\fBoutputChannel\fR] during evaluation |
| 713 | of the script(s) will be compared. Note that only output printed using |
| 714 | [\fB::puts\fR] is used for comparison. If \fB-output\fR is not specified, |
| 715 | output sent to \fBstdout\fR and [\fBoutputChannel\fR] is not processed for |
| 716 | comparison. |
| 717 | .TP |
| 718 | \fB-errorOutput \fIexpectedValue\fR |
| 719 | The \fB-errorOutput\fR attribute supplies the \fIexpectedValue\fR against |
| 720 | which any output sent to \fBstderr\fR or [\fBerrorChannel\fR] during |
| 721 | evaluation of the script(s) will be compared. Note that only output |
| 722 | printed using [\fB::puts\fR] is used for comparison. If \fB-errorOutput\fR |
| 723 | is not specified, output sent to \fBstderr\fR and [\fBerrorChannel\fR] is |
| 724 | not processed for comparison. |
| 725 | .TP |
| 726 | \fB-returnCodes \fIexpectedCodeList\fR |
| 727 | The optional \fB-returnCodes\fR attribute supplies \fIexpectedCodeList\fR, |
| 728 | a list of return codes that may be accepted from evaluation of the |
| 729 | \fB-body\fR script. If evaluation of the \fB-body\fR script returns |
| 730 | a code not in the \fIexpectedCodeList\fR, the test fails. All |
| 731 | return codes known to [\fBreturn\fR], in both numeric and symbolic |
| 732 | form, including extended return codes, are acceptable elements in |
| 733 | the \fIexpectedCodeList\fR. Default value is \fB{ok return}\fR. |
| 734 | .PP |
| 735 | To pass, a test must successfully evaluate its \fB-setup\fR, \fB-body\fR, |
| 736 | and \fB-cleanup\fR scripts. The return code of the \fB-body\fR script and |
| 737 | its result must match expected values, and if specified, output and error |
| 738 | data from the test must match expected \fB-output\fR and \fB-errorOutput\fR |
| 739 | values. If any of these conditions are not met, then the test fails. |
| 740 | Note that all scripts are evaluated in the context of the caller |
| 741 | of [\fBtest\fR]. |
| 742 | .PP |
| 743 | As long as [\fBtest\fR] is called with valid syntax and legal |
| 744 | values for all attributes, it will not raise an error. Test |
| 745 | failures are instead reported as output written to [\fBoutputChannel\fR]. |
| 746 | In default operation, a successful test produces no output. The output |
| 747 | messages produced by [\fBtest\fR] are controlled by the |
| 748 | [\fBconfigure -verbose\fR] option as described in \fBCONFIGURABLE OPTIONS\fR |
| 749 | below. Any output produced by the test scripts themselves should be |
| 750 | produced using [\fB::puts\fR] to [\fBoutputChannel\fR] or |
| 751 | [\fBerrorChannel\fR], so that users of the test suite may |
| 752 | easily capture output with the [\fBconfigure -outfile\fR] and |
| 753 | [\fBconfigure -errfile\fR] options, and so that the \fB-output\fR |
| 754 | and \fB-errorOutput\fR attributes work properly. |
| 755 | .SH "TEST CONSTRAINTS" |
| 756 | .PP |
| 757 | Constraints are used to determine whether or not a test should be skipped. |
| 758 | Each constraint has a name, which may be any string, and a boolean |
| 759 | value. Each [\fBtest\fR] has a \fB-constraints\fR value which is a |
| 760 | list of constraint names. There are two modes of constraint control. |
| 761 | Most frequently, the default mode is used, indicated by a setting |
| 762 | of [\fBconfigure -limitconstraints\fR] to false. The test will run |
| 763 | only if all constraints in the list are true-valued. Thus, |
| 764 | the \fB-constraints\fR option of [\fBtest\fR] is a convenient, symbolic |
| 765 | way to define any conditions required for the test to be possible or |
| 766 | meaningful. For example, a [\fBtest\fR] with \fB-constraints unix\fR |
| 767 | will only be run if the constraint \fBunix\fR is true, which indicates |
| 768 | the test suite is being run on a Unix platform. |
| 769 | .PP |
| 770 | Each [\fBtest\fR] should include whatever \fB-constraints\fR are |
| 771 | required to constrain it to run only where appropriate. Several |
| 772 | constraints are pre-defined in the \fBtcltest\fR package, listed |
| 773 | below. The registration of user-defined constraints is performed |
| 774 | by the [\fBtestConstraint\fR] command. User-defined constraints |
| 775 | may appear within a test file, or within the script specified |
| 776 | by the [\fBconfigure -load\fR] or [\fBconfigure -loadfile\fR] |
| 777 | options. |
| 778 | .PP |
| 779 | The following is a list of constraints pre-defined by the |
| 780 | \fBtcltest\fR package itself: |
| 781 | .TP |
| 782 | \fIsingleTestInterp\fR |
| 783 | test can only be run if all test files are sourced into a single interpreter |
| 784 | .TP |
| 785 | \fIunix\fR |
| 786 | test can only be run on any Unix platform |
| 787 | .TP |
| 788 | \fIwin\fR |
| 789 | test can only be run on any Windows platform |
| 790 | .TP |
| 791 | \fInt\fR |
| 792 | test can only be run on any Windows NT platform |
| 793 | .TP |
| 794 | \fI95\fR |
| 795 | test can only be run on any Windows 95 platform |
| 796 | .TP |
| 797 | \fI98\fR |
| 798 | test can only be run on any Windows 98 platform |
| 799 | .TP |
| 800 | \fImac\fR |
| 801 | test can only be run on any Mac platform |
| 802 | .TP |
| 803 | \fIunixOrWin\fR |
| 804 | test can only be run on a Unix or Windows platform |
| 805 | .TP |
| 806 | \fImacOrWin\fR |
| 807 | test can only be run on a Mac or Windows platform |
| 808 | .TP |
| 809 | \fImacOrUnix\fR |
| 810 | test can only be run on a Mac or Unix platform |
| 811 | .TP |
| 812 | \fItempNotWin\fR |
| 813 | test can not be run on Windows. This flag is used to temporarily |
| 814 | disable a test. |
| 815 | .TP |
| 816 | \fItempNotMac\fR |
| 817 | test can not be run on a Mac. This flag is used |
| 818 | to temporarily disable a test. |
| 819 | .TP |
| 820 | \fIunixCrash\fR |
| 821 | test crashes if it's run on Unix. This flag is used to temporarily |
| 822 | disable a test. |
| 823 | .TP |
| 824 | \fIwinCrash\fR |
| 825 | test crashes if it's run on Windows. This flag is used to temporarily |
| 826 | disable a test. |
| 827 | .TP |
| 828 | \fImacCrash\fR |
| 829 | test crashes if it's run on a Mac. This flag is used to temporarily |
| 830 | disable a test. |
| 831 | .TP |
| 832 | \fIemptyTest\fR |
| 833 | test is empty, and so not worth running, but it remains as a |
| 834 | place-holder for a test to be written in the future. This constraint |
| 835 | has value false to cause tests to be skipped unless the user specifies |
| 836 | otherwise. |
| 837 | .TP |
| 838 | \fIknownBug\fR |
| 839 | test is known to fail and the bug is not yet fixed. This constraint |
| 840 | has value false to cause tests to be skipped unless the user specifies |
| 841 | otherwise. |
| 842 | .TP |
| 843 | \fInonPortable\fR |
| 844 | test can only be run in some known development environment. |
| 845 | Some tests are inherently non-portable because they depend on things |
| 846 | like word length, file system configuration, window manager, etc. |
| 847 | This constraint has value false to cause tests to be skipped unless |
| 848 | the user specifies otherwise. |
| 849 | .TP |
| 850 | \fIuserInteraction\fR |
| 851 | test requires interaction from the user. This constraint has |
| 852 | value false to causes tests to be skipped unless the user specifies |
| 853 | otherwise. |
| 854 | .TP |
| 855 | \fIinteractive\fR |
| 856 | test can only be run in if the interpreter is in interactive mode |
| 857 | (when the global tcl_interactive variable is set to 1). |
| 858 | .TP |
| 859 | \fInonBlockFiles\fR |
| 860 | test can only be run if platform supports setting files into |
| 861 | nonblocking mode |
| 862 | .TP |
| 863 | \fIasyncPipeClose\fR |
| 864 | test can only be run if platform supports async flush and async close |
| 865 | on a pipe |
| 866 | .TP |
| 867 | \fIunixExecs\fR |
| 868 | test can only be run if this machine has Unix-style commands |
| 869 | \fBcat\fR, \fBecho\fR, \fBsh\fR, \fBwc\fR, \fBrm\fR, \fBsleep\fR, |
| 870 | \fBfgrep\fR, \fBps\fR, \fBchmod\fR, and \fBmkdir\fR available |
| 871 | .TP |
| 872 | \fIhasIsoLocale\fR |
| 873 | test can only be run if can switch to an ISO locale |
| 874 | .TP |
| 875 | \fIroot\fR |
| 876 | test can only run if Unix user is root |
| 877 | .TP |
| 878 | \fInotRoot\fR |
| 879 | test can only run if Unix user is not root |
| 880 | .TP |
| 881 | \fIeformat\fR |
| 882 | test can only run if app has a working version of sprintf with respect |
| 883 | to the "e" format of floating-point numbers. |
| 884 | .TP |
| 885 | \fIstdio\fR |
| 886 | test can only be run if [\fBinterpreter\fR] can be [\fBopen\fR]ed |
| 887 | as a pipe. |
| 888 | .PP |
| 889 | The alternative mode of constraint control is enabled by setting |
| 890 | [\fBconfigure -limitconstraints\fR] to true. With that configuration |
| 891 | setting, all existing constraints other than those in the constraint |
| 892 | list returned by [\fBconfigure -constraints\fR] are set to false. |
| 893 | When the value of [\fBconfigure -constraints\fR] |
| 894 | is set, all those constraints are set to true. The effect is that |
| 895 | when both options [\fBconfigure -constraints\fR] and |
| 896 | [\fBconfigure -limitconstraints\fR] are in use, only those tests including |
| 897 | only constraints from the [\fBconfigure -constraints\fR] list |
| 898 | are run; all others are skipped. For example, one might set |
| 899 | up a configuration with |
| 900 | .CS |
| 901 | \fBconfigure\fR -constraints knownBug \e |
| 902 | -limitconstraints true \e |
| 903 | -verbose pass |
| 904 | .CE |
| 905 | to run exactly those tests that exercise known bugs, and discover |
| 906 | whether any of them pass, indicating the bug had been fixed. |
| 907 | .SH "RUNNING ALL TESTS" |
| 908 | .PP |
| 909 | The single command [\fBrunAllTests\fR] is evaluated to run an entire |
| 910 | test suite, spanning many files and directories. The configuration |
| 911 | options of \fBtcltest\fR control the precise operations. The |
| 912 | [\fBrunAllTests\fR] command begins by printing a summary of its |
| 913 | configuration to [\fBoutputChannel\fR]. |
| 914 | .PP |
| 915 | Test files to be evaluated are sought in the directory |
| 916 | [\fBconfigure -testdir\fR]. The list of files in that directory |
| 917 | that match any of the patterns in [\fBconfigure -file\fR] and |
| 918 | match none of the patterns in [\fBconfigure -notfile\fR] is generated |
| 919 | and sorted. Then each file will be evaluated in turn. If |
| 920 | [\fBconfigure -singleproc\fR] is true, then each file will |
| 921 | be [\fBsource\fR]d in the caller's context. If it is false, |
| 922 | then a copy of [\fBinterpreter\fR] will be [\fBexec\fR]d to |
| 923 | evaluate each file. The multi-process operation is useful |
| 924 | when testing can cause errors so severe that a process |
| 925 | terminates. Although such an error may terminate a child |
| 926 | process evaluating one file, the master process can continue |
| 927 | with the rest of the test suite. In multi-process operation, |
| 928 | the configuration of \fBtcltest\fR in the master process is |
| 929 | passed to the child processes as command line arguments, |
| 930 | with the exception of [\fBconfigure -outfile\fR]. The |
| 931 | [\fBrunAllTests\fR] command in the |
| 932 | master process collects all output from the child processes |
| 933 | and collates their results into one master report. Any |
| 934 | reports of individual test failures, or messages requested |
| 935 | by a [\fBconfigure -verbose\fR] setting are passed directly |
| 936 | on to [\fBoutputChannel\fR] by the master process. |
| 937 | .PP |
| 938 | After evaluating all selected test files, a summary of the |
| 939 | results is printed to [\fBoutputChannel\fR]. The summary |
| 940 | includes the total number of [\fBtest\fR]s evaluated, broken |
| 941 | down into those skipped, those passed, and those failed. |
| 942 | The summary also notes the number of files evaluated, and the names |
| 943 | of any files with failing tests or errors. A list of |
| 944 | the constraints that caused tests to be skipped, and the |
| 945 | number of tests skipped for each is also printed. Also, |
| 946 | messages are printed if it appears that evaluation of |
| 947 | a test file has caused any temporary files to be left |
| 948 | behind in [\fBconfigure -tmpdir\fR]. |
| 949 | .PP |
| 950 | Having completed and summarized all selected test files, |
| 951 | [\fBrunAllTests\fR] then recursively acts on subdirectories |
| 952 | of [\fBconfigure -testdir\fR]. All subdirectories that |
| 953 | match any of the patterns in [\fBconfigure -relateddir\fR] |
| 954 | and do not match any of the patterns in |
| 955 | [\fBconfigure -asidefromdir\fR] are examined. If |
| 956 | a file named \fBall.tcl\fR is found in such a directory, |
| 957 | it will be [\fBsource\fR]d in the caller's context. |
| 958 | Whether or not an examined directory contains an |
| 959 | \fBall.tcl\fR file, its subdirectories are also scanned |
| 960 | against the [\fBconfigure -relateddir\fR] and |
| 961 | [\fBconfigure -asidefromdir\fR] patterns. In this way, |
| 962 | many directories in a directory tree can have all their |
| 963 | test files evaluated by a single [\fBrunAllTests\fR] |
| 964 | command. |
| 965 | .SH "CONFIGURABLE OPTIONS" |
| 966 | The [\fBconfigure\fR] command is used to set and query the configurable |
| 967 | options of \fBtcltest\fR. The valid options are: |
| 968 | .TP |
| 969 | \fB-singleproc \fIboolean\fR |
| 970 | Controls whether or not [\fBrunAllTests\fR] spawns a child process for |
| 971 | each test file. No spawning when \fIboolean\fR is true. Default |
| 972 | value is false. |
| 973 | .TP |
| 974 | \fB-debug \fIlevel\fR |
| 975 | Sets the debug level to \fIlevel\fR, an integer value indicating how |
| 976 | much debugging information should be printed to stdout. Note that |
| 977 | debug messages always go to stdout, independent of the value of |
| 978 | [\fBconfigure -outfile\fR]. Default value is 0. Levels are defined as: |
| 979 | .RS |
| 980 | .IP 0 |
| 981 | Do not display any debug information. |
| 982 | .IP 1 |
| 983 | Display information regarding whether a test is skipped because it |
| 984 | doesn't match any of the tests that were specified using by |
| 985 | [\fBconfigure -match\fR] (userSpecifiedNonMatch) or matches any of |
| 986 | the tests specified by [\fBconfigure -skip\fR] (userSpecifiedSkip). Also |
| 987 | print warnings about possible lack of cleanup or balance in test files. |
| 988 | Also print warnings about any re-use of test names. |
| 989 | .IP 2 |
| 990 | Display the flag array parsed by the command line processor, the |
| 991 | contents of the ::env array, and all user-defined variables that exist |
| 992 | in the current namespace as they are used. |
| 993 | .IP 3 |
| 994 | Display information regarding what individual procs in the test |
| 995 | harness are doing. |
| 996 | .RE |
| 997 | .TP |
| 998 | \fB-verbose \fIlevel\fR |
| 999 | Sets the type of output verbosity desired to \fIlevel\fR, |
| 1000 | a list of zero or more of the elements \fBbody\fR, \fBpass\fR, |
| 1001 | \fBskip\fR, \fBstart\fR, and \fBerror\fR. Default value |
| 1002 | is \fB{body error}\fR. |
| 1003 | Levels are defined as: |
| 1004 | .RS |
| 1005 | .IP "body (b)" |
| 1006 | Display the body of failed tests |
| 1007 | .IP "pass (p)" |
| 1008 | Print output when a test passes |
| 1009 | .IP "skip (s)" |
| 1010 | Print output when a test is skipped |
| 1011 | .IP "start (t)" |
| 1012 | Print output whenever a test starts |
| 1013 | .IP "error (e)" |
| 1014 | Print errorInfo and errorCode, if they exist, when a test return code |
| 1015 | does not match its expected return code |
| 1016 | .RE |
| 1017 | The single letter abbreviations noted above are also recognized |
| 1018 | so that [\fBconfigure -verbose pt\fR] is the same as |
| 1019 | [\fBconfigure -verbose {pass start}\fR]. |
| 1020 | .TP |
| 1021 | \fB-preservecore \fIlevel\fR |
| 1022 | Sets the core preservation level to \fIlevel\fR. This level |
| 1023 | determines how stringent checks for core files are. Default |
| 1024 | value is 0. Levels are defined as: |
| 1025 | .RS |
| 1026 | .IP 0 |
| 1027 | No checking - do not check for core files at the end of each test |
| 1028 | command, but do check for them in [\fBrunAllTests\fR] after all |
| 1029 | test files have been evaluated. |
| 1030 | .IP 1 |
| 1031 | Also check for core files at the end of each [\fBtest\fR] command. |
| 1032 | .IP 2 |
| 1033 | Check for core files at all times described above, and save a |
| 1034 | copy of each core file produced in [\fBconfigure -tmpdir\fR]. |
| 1035 | .RE |
| 1036 | .TP |
| 1037 | \fB-limitconstraints \fIboolean\fR |
| 1038 | Sets the mode by which [\fBtest\fR] honors constraints as described |
| 1039 | in \fBTESTS\fR above. Default value is false. |
| 1040 | .TP |
| 1041 | \fB-constraints \fIlist\fR |
| 1042 | Sets all the constraints in \fIlist\fR to true. Also used in |
| 1043 | combination with [\fBconfigure -limitconstraints true\fR] to control an |
| 1044 | alternative constraint mode as described in \fBTESTS\fR above. |
| 1045 | Default value is an empty list. |
| 1046 | .TP |
| 1047 | \fB-tmpdir \fIdirectory\fR |
| 1048 | Sets the temporary directory to be used by [\fBmakeFile\fR], |
| 1049 | [\fBmakeDirectory\fR], [\fBviewFile\fR], [\fBremoveFile\fR], |
| 1050 | and [\fBremoveDirectory\fR] as the default directory where |
| 1051 | temporary files and directories created by test files should |
| 1052 | be created. Default value is [\fBworkingDirectory\fR]. |
| 1053 | .TP |
| 1054 | \fB-testdir \fIdirectory\fR |
| 1055 | Sets the directory searched by [\fBrunAllTests\fR] for test files |
| 1056 | and subdirectories. Default value is [\fBworkingDirectory\fR]. |
| 1057 | .TP |
| 1058 | \fB-file \fIpatternList\fR |
| 1059 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine |
| 1060 | what test files to evaluate. Default value is \fB*.test\fR. |
| 1061 | .TP |
| 1062 | \fB-notfile \fIpatternList\fR |
| 1063 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine |
| 1064 | what test files to skip. Default value is \fBl.*.test\fR, so |
| 1065 | that any SCCS lock files are skipped. |
| 1066 | .TP |
| 1067 | \fB-relateddir \fIpatternList\fR |
| 1068 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine |
| 1069 | what subdirectories to search for an \fBall.tcl\fR file. Default |
| 1070 | value is \fB*\fR. |
| 1071 | .TP |
| 1072 | \fB-asidefromdir \fIpatternList\fR |
| 1073 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine |
| 1074 | what subdirectories to skip when searching for an \fBall.tcl\fR file. |
| 1075 | Default value is an empty list. |
| 1076 | .TP |
| 1077 | \fB-match \fIpatternList\fR |
| 1078 | Set the list of patterns used by [\fBtest\fR] to determine whether |
| 1079 | a test should be run. Default value is \fB*\fR. |
| 1080 | .TP |
| 1081 | \fB-skip \fIpatternList\fR |
| 1082 | Set the list of patterns used by [\fBtest\fR] to determine whether |
| 1083 | a test should be skipped. Default value is an empty list. |
| 1084 | .TP |
| 1085 | \fB-load \fIscript\fR |
| 1086 | Sets a script to be evaluated by [\fBloadTestedCommands\fR]. |
| 1087 | Default value is an empty script. |
| 1088 | .TP |
| 1089 | \fB-loadfile \fIfilename\fR |
| 1090 | Sets the filename from which to read a script to be evaluated |
| 1091 | by [\fBloadTestedCommands\fR]. This is an alternative to |
| 1092 | \fB-load\fR. They cannot be used together. |
| 1093 | .TP |
| 1094 | \fB-outfile \fIfilename\fR |
| 1095 | Sets the file to which all output produced by tcltest should be |
| 1096 | written. A file named \fIfilename\fR will be [\fBopen\fR]ed for writing, |
| 1097 | and the resulting channel will be set as the value of [\fBoutputChannel\fR]. |
| 1098 | .TP |
| 1099 | \fB-errfile \fIfilename\fR |
| 1100 | Sets the file to which all error output produced by tcltest |
| 1101 | should be written. A file named \fIfilename\fR will be [\fBopen\fR]ed |
| 1102 | for writing, and the resulting channel will be set as the value |
| 1103 | of [\fBerrorChannel\fR]. |
| 1104 | .SH "CREATING TEST SUITES WITH TCLTEST" |
| 1105 | .PP |
| 1106 | The fundamental element of a test suite is the individual [\fBtest\fR] |
| 1107 | command. We begin with several examples. |
| 1108 | .IP [1] |
| 1109 | Test of a script that returns normally. |
| 1110 | .CS |
| 1111 | \fBtest\fR example-1.0 {normal return} { |
| 1112 | format %s value |
| 1113 | } value |
| 1114 | .CE |
| 1115 | .IP [2] |
| 1116 | Test of a script that requires context setup and cleanup. Note the |
| 1117 | bracing and indenting style that avoids any need for line continuation. |
| 1118 | .CS |
| 1119 | \fBtest\fR example-1.1 {test file existence} -setup { |
| 1120 | set file [makeFile {} test] |
| 1121 | } -body { |
| 1122 | file exists $file |
| 1123 | } -cleanup { |
| 1124 | removeFile test |
| 1125 | } -result 1 |
| 1126 | .CE |
| 1127 | .IP [3] |
| 1128 | Test of a script that raises an error. |
| 1129 | .CS |
| 1130 | \fBtest\fR example-1.2 {error return} -body { |
| 1131 | error message |
| 1132 | } -returnCodes error -result message |
| 1133 | .CE |
| 1134 | .IP [4] |
| 1135 | Test with a constraint. |
| 1136 | .CS |
| 1137 | \fBtest\fR example-1.3 {user owns created files} -constraints { |
| 1138 | unix |
| 1139 | } -setup { |
| 1140 | set file [makeFile {} test] |
| 1141 | } -body { |
| 1142 | file attributes $file -owner |
| 1143 | } -cleanup { |
| 1144 | removeFile test |
| 1145 | } -result $::tcl_platform(user) |
| 1146 | .CE |
| 1147 | .PP |
| 1148 | At the next higher layer of organization, several [\fBtest\fR] commands |
| 1149 | are gathered together into a single test file. Test files should have |
| 1150 | names with the \fB.test\fR extension, because that is the default pattern |
| 1151 | used by [\fBrunAllTests\fR] to find test files. It is a good rule of |
| 1152 | thumb to have one test file for each source code file of your project. |
| 1153 | It is good practice to edit the test file and the source code file |
| 1154 | together, keeping tests synchronized with code changes. |
| 1155 | .PP |
| 1156 | Most of the code in the test file should be the [\fBtest\fR] commands. |
| 1157 | Use constraints to skip tests, rather than conditional evaluation |
| 1158 | of [\fBtest\fR]. That is, do this: |
| 1159 | .IP [5] |
| 1160 | .CS |
| 1161 | \fBtestConstraint\fR X [expr $myRequirement] |
| 1162 | \fBtest\fR goodConditionalTest {} X { |
| 1163 | # body |
| 1164 | } result |
| 1165 | .CE |
| 1166 | and do not do this: |
| 1167 | .IP [6] |
| 1168 | .CS |
| 1169 | if $myRequirement { |
| 1170 | test badConditionalTest {} { |
| 1171 | #body |
| 1172 | } result |
| 1173 | } |
| 1174 | .CE |
| 1175 | .PP |
| 1176 | Use the \fB-setup\fR and \fB-cleanup\fR options to establish and release |
| 1177 | all context requirements of the test body. Do not make tests depend on |
| 1178 | prior tests in the file. Those prior tests might be skipped. If several |
| 1179 | consecutive tests require the same context, the appropriate setup |
| 1180 | and cleanup scripts may be stored in variable for passing to each tests |
| 1181 | \fB-setup\fR and \fB-cleanup\fR options. This is a better solution than |
| 1182 | performing setup outside of [\fBtest\fR] commands, because the setup will |
| 1183 | only be done if necessary, and any errors during setup will be reported, |
| 1184 | and not cause the test file to abort. |
| 1185 | .PP |
| 1186 | A test file should be able to be combined with other test files and not |
| 1187 | interfere with them, even when [\fBconfigure -singleproc 1\fR] causes |
| 1188 | all files to be evaluated in a common interpreter. A simple way to |
| 1189 | achieve this is to have your tests define all their commands and variables |
| 1190 | in a namespace that is deleted when the test file evaluation is complete. |
| 1191 | A good namespace to use is a child namespace \fBtest\fR of the namespace |
| 1192 | of the module you are testing. |
| 1193 | .PP |
| 1194 | A test file should also be able to be evaluated directly as a script, |
| 1195 | not depending on being called by a master [\fBrunAllTests\fR]. This |
| 1196 | means that each test file should process command line arguments to give |
| 1197 | the tester all the configuration control that \fBtcltest\fR provides. |
| 1198 | .PP |
| 1199 | After all [\fBtest\fR]s in a test file, the command [\fBcleanupTests\fR] |
| 1200 | should be called. |
| 1201 | .IP [7] |
| 1202 | Here is a sketch of a sample test file illustrating those points: |
| 1203 | .CS |
| 1204 | package require tcltest 2.2 |
| 1205 | eval \fB::tcltest::configure\fR $argv |
| 1206 | package require example |
| 1207 | namespace eval ::example::test { |
| 1208 | namespace import ::tcltest::* |
| 1209 | \fBtestConstraint\fR X [expr {...}] |
| 1210 | variable SETUP {#common setup code} |
| 1211 | variable CLEANUP {#common cleanup code} |
| 1212 | \fBtest\fR example-1 {} -setup $SETUP -body { |
| 1213 | # First test |
| 1214 | } -cleanup $CLEANUP -result {...} |
| 1215 | \fBtest\fR example-2 {} -constraints X -setup $SETUP -body { |
| 1216 | # Second test; constrained |
| 1217 | } -cleanup $CLEANUP -result {...} |
| 1218 | \fBtest\fR example-3 {} { |
| 1219 | # Third test; no context required |
| 1220 | } {...} |
| 1221 | \fBcleanupTests\fR |
| 1222 | } |
| 1223 | namespace delete ::example::test |
| 1224 | .CE |
| 1225 | .PP |
| 1226 | The next level of organization is a full test suite, made up of several |
| 1227 | test files. One script is used to control the entire suite. The |
| 1228 | basic function of this script is to call [\fBrunAllTests\fR] after |
| 1229 | doing any necessary setup. This script is usually named \fBall.tcl\fR |
| 1230 | because that's the default name used by [\fBrunAllTests\fR] when combining |
| 1231 | multiple test suites into one testing run. |
| 1232 | .IP [8] |
| 1233 | Here is a sketch of a sample test suite master script: |
| 1234 | .CS |
| 1235 | package require Tcl 8.4 |
| 1236 | package require tcltest 2.2 |
| 1237 | package require example |
| 1238 | \fB::tcltest::configure\fR -testdir \ |
| 1239 | [file dirname [file normalize [info script]]] |
| 1240 | eval \fB::tcltest::configure\fR $argv |
| 1241 | \fB::tcltest::runAllTests\fR |
| 1242 | .CE |
| 1243 | .SH COMPATIBILITY |
| 1244 | .PP |
| 1245 | A number of commands and variables in the \fB::tcltest\fR namespace |
| 1246 | provided by earlier releases of \fBtcltest\fR have not been documented |
| 1247 | here. They are no longer part of the supported public interface of |
| 1248 | \fBtcltest\fR and should not be used in new test suites. However, |
| 1249 | to continue to support existing test suites written to the older |
| 1250 | interface specifications, many of those deprecated commands and |
| 1251 | variables still work as before. For example, in many circumstances, |
| 1252 | [\fBconfigure\fR] will be automatically called shortly after |
| 1253 | [\fBpackage require tcltest 2.1\fR] succeeds with arguments |
| 1254 | from the variable \fB::argv\fR. This is to support test suites |
| 1255 | that depend on the old behavior that \fBtcltest\fR was automatically |
| 1256 | configured from command line arguments. New test files should not |
| 1257 | depend on this, but should explicitly include |
| 1258 | .CS |
| 1259 | eval \fB::tcltest::configure\fR $::argv |
| 1260 | .CE |
| 1261 | to establish a configuration from command line arguments. |
| 1262 | .SH "KNOWN ISSUES" |
| 1263 | There are two known issues related to nested evaluations of [\fBtest\fR]. |
| 1264 | The first issue relates to the stack level in which test scripts are |
| 1265 | executed. Tests nested within other tests may be executed at the same |
| 1266 | stack level as the outermost test. For example, in the following code: |
| 1267 | .CS |
| 1268 | \fBtest\fR level-1.1 {level 1} { |
| 1269 | -body { |
| 1270 | \fBtest\fR level-2.1 {level 2} { |
| 1271 | } |
| 1272 | } |
| 1273 | } |
| 1274 | .CE |
| 1275 | any script executed in level-2.1 may be executed at the same stack |
| 1276 | level as the script defined for level-1.1. |
| 1277 | .PP |
| 1278 | In addition, while two [\fBtest\fR]s have been run, results will only |
| 1279 | be reported by [\fBcleanupTests\fR] for tests at the same level as |
| 1280 | test level-1.1. However, test results for all tests run prior to |
| 1281 | level-1.1 will be available when test level-2.1 runs. What this |
| 1282 | means is that if you try to access the test results for test level-2.1, |
| 1283 | it will may say that 'm' tests have run, 'n' tests have |
| 1284 | been skipped, 'o' tests have passed and 'p' tests have failed, |
| 1285 | where 'm', 'n', 'o', and 'p' refer to tests that were run at the |
| 1286 | same test level as test level-1.1. |
| 1287 | .PP |
| 1288 | Implementation of output and error comparison in the test command |
| 1289 | depends on usage of ::puts in your application code. Output is |
| 1290 | intercepted by redefining the ::puts command while the defined test |
| 1291 | script is being run. Errors thrown by C procedures or printed |
| 1292 | directly from C applications will not be caught by the test command. |
| 1293 | Therefore, usage of the \fB-output\fR and \fB-errorOuput\fR |
| 1294 | options to [\fBtest\fR] is useful only for pure Tcl applications |
| 1295 | that use [\fB::puts\fR] to produce output. |
| 1296 | |
| 1297 | .SH KEYWORDS |
| 1298 | test, test harness, test suite |