Commit | Line | Data |
---|---|---|
920dae64 AT |
1 | '\" |
2 | '\" Copyright (c) 1990-1994 The Regents of the University of California | |
3 | '\" Copyright (c) 1994-1997 Sun Microsystems, Inc. | |
4 | '\" Copyright (c) 1998-1999 Scriptics Corporation | |
5 | '\" Copyright (c) 2000 Ajuba Solutions | |
6 | '\" Contributions from Don Porter, NIST, 2002. (not subject to US copyright) | |
7 | '\" | |
8 | '\" See the file "license.terms" for information on usage and redistribution | |
9 | '\" of this file, and for a DISCLAIMER OF ALL WARRANTIES. | |
10 | '\" | |
11 | '\" RCS: @(#) $Id: tcltest.n,v 1.38.2.5 2004/10/27 14:43:13 dkf Exp $ | |
12 | '\" | |
13 | '\" The definitions below are for supplemental macros used in Tcl/Tk | |
14 | '\" manual entries. | |
15 | '\" | |
16 | '\" .AP type name in/out ?indent? | |
17 | '\" Start paragraph describing an argument to a library procedure. | |
18 | '\" type is type of argument (int, etc.), in/out is either "in", "out", | |
19 | '\" or "in/out" to describe whether procedure reads or modifies arg, | |
20 | '\" and indent is equivalent to second arg of .IP (shouldn't ever be | |
21 | '\" needed; use .AS below instead) | |
22 | '\" | |
23 | '\" .AS ?type? ?name? | |
24 | '\" Give maximum sizes of arguments for setting tab stops. Type and | |
25 | '\" name are examples of largest possible arguments that will be passed | |
26 | '\" to .AP later. If args are omitted, default tab stops are used. | |
27 | '\" | |
28 | '\" .BS | |
29 | '\" Start box enclosure. From here until next .BE, everything will be | |
30 | '\" enclosed in one large box. | |
31 | '\" | |
32 | '\" .BE | |
33 | '\" End of box enclosure. | |
34 | '\" | |
35 | '\" .CS | |
36 | '\" Begin code excerpt. | |
37 | '\" | |
38 | '\" .CE | |
39 | '\" End code excerpt. | |
40 | '\" | |
41 | '\" .VS ?version? ?br? | |
42 | '\" Begin vertical sidebar, for use in marking newly-changed parts | |
43 | '\" of man pages. The first argument is ignored and used for recording | |
44 | '\" the version when the .VS was added, so that the sidebars can be | |
45 | '\" found and removed when they reach a certain age. If another argument | |
46 | '\" is present, then a line break is forced before starting the sidebar. | |
47 | '\" | |
48 | '\" .VE | |
49 | '\" End of vertical sidebar. | |
50 | '\" | |
51 | '\" .DS | |
52 | '\" Begin an indented unfilled display. | |
53 | '\" | |
54 | '\" .DE | |
55 | '\" End of indented unfilled display. | |
56 | '\" | |
57 | '\" .SO | |
58 | '\" Start of list of standard options for a Tk widget. The | |
59 | '\" options follow on successive lines, in four columns separated | |
60 | '\" by tabs. | |
61 | '\" | |
62 | '\" .SE | |
63 | '\" End of list of standard options for a Tk widget. | |
64 | '\" | |
65 | '\" .OP cmdName dbName dbClass | |
66 | '\" Start of description of a specific option. cmdName gives the | |
67 | '\" option's name as specified in the class command, dbName gives | |
68 | '\" the option's name in the option database, and dbClass gives | |
69 | '\" the option's class in the option database. | |
70 | '\" | |
71 | '\" .UL arg1 arg2 | |
72 | '\" Print arg1 underlined, then print arg2 normally. | |
73 | '\" | |
74 | '\" RCS: @(#) $Id: man.macros,v 1.4 2000/08/25 06:18:32 ericm Exp $ | |
75 | '\" | |
76 | '\" # Set up traps and other miscellaneous stuff for Tcl/Tk man pages. | |
77 | .if t .wh -1.3i ^B | |
78 | .nr ^l \n(.l | |
79 | .ad b | |
80 | '\" # Start an argument description | |
81 | .de AP | |
82 | .ie !"\\$4"" .TP \\$4 | |
83 | .el \{\ | |
84 | . ie !"\\$2"" .TP \\n()Cu | |
85 | . el .TP 15 | |
86 | .\} | |
87 | .ta \\n()Au \\n()Bu | |
88 | .ie !"\\$3"" \{\ | |
89 | \&\\$1 \\fI\\$2\\fP (\\$3) | |
90 | .\".b | |
91 | .\} | |
92 | .el \{\ | |
93 | .br | |
94 | .ie !"\\$2"" \{\ | |
95 | \&\\$1 \\fI\\$2\\fP | |
96 | .\} | |
97 | .el \{\ | |
98 | \&\\fI\\$1\\fP | |
99 | .\} | |
100 | .\} | |
101 | .. | |
102 | '\" # define tabbing values for .AP | |
103 | .de AS | |
104 | .nr )A 10n | |
105 | .if !"\\$1"" .nr )A \\w'\\$1'u+3n | |
106 | .nr )B \\n()Au+15n | |
107 | .\" | |
108 | .if !"\\$2"" .nr )B \\w'\\$2'u+\\n()Au+3n | |
109 | .nr )C \\n()Bu+\\w'(in/out)'u+2n | |
110 | .. | |
111 | .AS Tcl_Interp Tcl_CreateInterp in/out | |
112 | '\" # BS - start boxed text | |
113 | '\" # ^y = starting y location | |
114 | '\" # ^b = 1 | |
115 | .de BS | |
116 | .br | |
117 | .mk ^y | |
118 | .nr ^b 1u | |
119 | .if n .nf | |
120 | .if n .ti 0 | |
121 | .if n \l'\\n(.lu\(ul' | |
122 | .if n .fi | |
123 | .. | |
124 | '\" # BE - end boxed text (draw box now) | |
125 | .de BE | |
126 | .nf | |
127 | .ti 0 | |
128 | .mk ^t | |
129 | .ie n \l'\\n(^lu\(ul' | |
130 | .el \{\ | |
131 | .\" Draw four-sided box normally, but don't draw top of | |
132 | .\" box if the box started on an earlier page. | |
133 | .ie !\\n(^b-1 \{\ | |
134 | \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul' | |
135 | .\} | |
136 | .el \}\ | |
137 | \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul' | |
138 | .\} | |
139 | .\} | |
140 | .fi | |
141 | .br | |
142 | .nr ^b 0 | |
143 | .. | |
144 | '\" # VS - start vertical sidebar | |
145 | '\" # ^Y = starting y location | |
146 | '\" # ^v = 1 (for troff; for nroff this doesn't matter) | |
147 | .de VS | |
148 | .if !"\\$2"" .br | |
149 | .mk ^Y | |
150 | .ie n 'mc \s12\(br\s0 | |
151 | .el .nr ^v 1u | |
152 | .. | |
153 | '\" # VE - end of vertical sidebar | |
154 | .de VE | |
155 | .ie n 'mc | |
156 | .el \{\ | |
157 | .ev 2 | |
158 | .nf | |
159 | .ti 0 | |
160 | .mk ^t | |
161 | \h'|\\n(^lu+3n'\L'|\\n(^Yu-1v\(bv'\v'\\n(^tu+1v-\\n(^Yu'\h'-|\\n(^lu+3n' | |
162 | .sp -1 | |
163 | .fi | |
164 | .ev | |
165 | .\} | |
166 | .nr ^v 0 | |
167 | .. | |
168 | '\" # Special macro to handle page bottom: finish off current | |
169 | '\" # box/sidebar if in box/sidebar mode, then invoked standard | |
170 | '\" # page bottom macro. | |
171 | .de ^B | |
172 | .ev 2 | |
173 | 'ti 0 | |
174 | 'nf | |
175 | .mk ^t | |
176 | .if \\n(^b \{\ | |
177 | .\" Draw three-sided box if this is the box's first page, | |
178 | .\" draw two sides but no top otherwise. | |
179 | .ie !\\n(^b-1 \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c | |
180 | .el \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c | |
181 | .\} | |
182 | .if \\n(^v \{\ | |
183 | .nr ^x \\n(^tu+1v-\\n(^Yu | |
184 | \kx\h'-\\nxu'\h'|\\n(^lu+3n'\ky\L'-\\n(^xu'\v'\\n(^xu'\h'|0u'\c | |
185 | .\} | |
186 | .bp | |
187 | 'fi | |
188 | .ev | |
189 | .if \\n(^b \{\ | |
190 | .mk ^y | |
191 | .nr ^b 2 | |
192 | .\} | |
193 | .if \\n(^v \{\ | |
194 | .mk ^Y | |
195 | .\} | |
196 | .. | |
197 | '\" # DS - begin display | |
198 | .de DS | |
199 | .RS | |
200 | .nf | |
201 | .sp | |
202 | .. | |
203 | '\" # DE - end display | |
204 | .de DE | |
205 | .fi | |
206 | .RE | |
207 | .sp | |
208 | .. | |
209 | '\" # SO - start of list of standard options | |
210 | .de SO | |
211 | .SH "STANDARD OPTIONS" | |
212 | .LP | |
213 | .nf | |
214 | .ta 5.5c 11c | |
215 | .ft B | |
216 | .. | |
217 | '\" # SE - end of list of standard options | |
218 | .de SE | |
219 | .fi | |
220 | .ft R | |
221 | .LP | |
222 | See the \\fBoptions\\fR manual entry for details on the standard options. | |
223 | .. | |
224 | '\" # OP - start of full description for a single option | |
225 | .de OP | |
226 | .LP | |
227 | .nf | |
228 | .ta 4c | |
229 | Command-Line Name: \\fB\\$1\\fR | |
230 | Database Name: \\fB\\$2\\fR | |
231 | Database Class: \\fB\\$3\\fR | |
232 | .fi | |
233 | .IP | |
234 | .. | |
235 | '\" # CS - begin code excerpt | |
236 | .de CS | |
237 | .RS | |
238 | .nf | |
239 | .ta .25i .5i .75i 1i | |
240 | .. | |
241 | '\" # CE - end code excerpt | |
242 | .de CE | |
243 | .fi | |
244 | .RE | |
245 | .. | |
246 | .de UL | |
247 | \\$1\l'|0\(ul'\\$2 | |
248 | .. | |
249 | .TH "tcltest" n 2.2 tcltest "Tcl Bundled Packages" | |
250 | .BS | |
251 | '\" Note: do not modify the .SH NAME line immediately below! | |
252 | .SH NAME | |
253 | tcltest \- Test harness support code and utilities | |
254 | .SH SYNOPSIS | |
255 | .nf | |
256 | \fBpackage require tcltest ?2.2.5?\fR | |
257 | .sp | |
258 | \fBtcltest::test \fIname description ?option value ...?\fR | |
259 | \fBtcltest::test \fIname description ?constraints? body result\fR | |
260 | .sp | |
261 | \fBtcltest::loadTestedCommands\fR | |
262 | \fBtcltest::makeDirectory \fIname ?directory?\fR | |
263 | \fBtcltest::removeDirectory \fIname ?directory?\fR | |
264 | \fBtcltest::makeFile \fIcontents name ?directory?\fR | |
265 | \fBtcltest::removeFile \fIname ?directory?\fR | |
266 | \fBtcltest::viewFile \fIname ?directory?\fR | |
267 | \fBtcltest::cleanupTests \fI?runningMultipleTests?\fR | |
268 | \fBtcltest::runAllTests\fR | |
269 | .sp | |
270 | \fBtcltest::configure\fR | |
271 | \fBtcltest::configure \fIoption\fR | |
272 | \fBtcltest::configure \fIoption value ?option value ...?\fR | |
273 | \fBtcltest::customMatch \fImode command\fR | |
274 | \fBtcltest::testConstraint \fIconstraint ?value?\fR | |
275 | \fBtcltest::outputChannel \fI?channelID?\fR | |
276 | \fBtcltest::errorChannel \fI?channelID?\fR | |
277 | \fBtcltest::interpreter \fI?interp?\fR | |
278 | .sp | |
279 | \fBtcltest::debug \fI?level?\fR | |
280 | \fBtcltest::errorFile \fI?filename?\fR | |
281 | \fBtcltest::limitConstraints \fI?boolean?\fR | |
282 | \fBtcltest::loadFile \fI?filename?\fR | |
283 | \fBtcltest::loadScript \fI?script?\fR | |
284 | \fBtcltest::match \fI?patternList?\fR | |
285 | \fBtcltest::matchDirectories \fI?patternList?\fR | |
286 | \fBtcltest::matchFiles \fI?patternList?\fR | |
287 | \fBtcltest::outputFile \fI?filename?\fR | |
288 | \fBtcltest::preserveCore \fI?level?\fR | |
289 | \fBtcltest::singleProcess \fI?boolean?\fR | |
290 | \fBtcltest::skip \fI?patternList?\fR | |
291 | \fBtcltest::skipDirectories \fI?patternList?\fR | |
292 | \fBtcltest::skipFiles \fI?patternList?\fR | |
293 | \fBtcltest::temporaryDirectory \fI?directory?\fR | |
294 | \fBtcltest::testsDirectory \fI?directory?\fR | |
295 | \fBtcltest::verbose \fI?level?\fR | |
296 | .sp | |
297 | \fBtcltest::test \fIname description optionList\fR | |
298 | \fBtcltest::bytestring \fIstring\fR | |
299 | \fBtcltest::normalizeMsg \fImsg\fR | |
300 | \fBtcltest::normalizePath \fIpathVar\fR | |
301 | \fBtcltest::workingDirectory \fI?dir?\fR | |
302 | .fi | |
303 | .BE | |
304 | .SH DESCRIPTION | |
305 | .PP | |
306 | The \fBtcltest\fR package provides several utility commands useful | |
307 | in the construction of test suites for code instrumented to be | |
308 | run by evaluation of Tcl commands. Notably the built-in commands | |
309 | of the Tcl library itself are tested by a test suite using the | |
310 | tcltest package. | |
311 | .PP | |
312 | All the commands provided by the \fBtcltest\fR package are defined | |
313 | in and exported from the \fB::tcltest\fR namespace, as indicated in | |
314 | the \fBSYNOPSIS\fR above. In the following sections, all commands | |
315 | will be described by their simple names, in the interest of brevity. | |
316 | .PP | |
317 | The central command of \fBtcltest\fR is [\fBtest\fR] that defines | |
318 | and runs a test. Testing with [\fBtest\fR] involves evaluation | |
319 | of a Tcl script and comparing the result to an expected result, as | |
320 | configured and controlled by a number of options. Several other | |
321 | commands provided by \fBtcltest\fR govern the configuration of | |
322 | [\fBtest\fR] and the collection of many [\fBtest\fR] commands into | |
323 | test suites. | |
324 | .PP | |
325 | See \fBCREATING TEST SUITES WITH TCLTEST\fR below for an extended example | |
326 | of how to use the commands of \fBtcltest\fR to produce test suites | |
327 | for your Tcl-enabled code. | |
328 | .SH COMMANDS | |
329 | .TP | |
330 | \fBtest\fR \fIname description ?option value ...?\fR | |
331 | Defines and possibly runs a test with the name \fIname\fR and | |
332 | description \fIdescription\fR. The name and description of a test | |
333 | are used in messages reported by [\fBtest\fR] during the | |
334 | test, as configured by the options of \fBtcltest\fR. The | |
335 | remaining \fIoption value\fR arguments to [\fBtest\fR] | |
336 | define the test, including the scripts to run, the conditions | |
337 | under which to run them, the expected result, and the means | |
338 | by which the expected and actual results should be compared. | |
339 | See \fBTESTS\fR below for a complete description of the valid | |
340 | options and how they define a test. The [\fBtest\fR] command | |
341 | returns an empty string. | |
342 | .TP | |
343 | \fBtest\fR \fIname description ?constraints? body result\fR | |
344 | This form of [\fBtest\fR] is provided to support test suites written | |
345 | for version 1 of the \fBtcltest\fR package, and also a simpler | |
346 | interface for a common usage. It is the same as | |
347 | [\fBtest\fR \fIname description\fB -constraints \fIconstraints\fB -body | |
348 | \fIbody\fB -result \fIresult\fR]. All other options to [\fBtest\fR] | |
349 | take their default values. When \fIconstraints\fR is omitted, this | |
350 | form of [\fBtest\fR] can be distinguished from the first because | |
351 | all \fIoption\fRs begin with ``-''. | |
352 | .TP | |
353 | \fBloadTestedCommands\fR | |
354 | Evaluates in the caller's context the script specified by | |
355 | [\fBconfigure -load\fR] or [\fBconfigure -loadfile\fR]. | |
356 | Returns the result of that script evaluation, including any error | |
357 | raised by the script. Use this command and the related | |
358 | configuration options to provide the commands to be tested to | |
359 | the interpreter running the test suite. | |
360 | .TP | |
361 | \fBmakeFile\fR \fIcontents name ?directory?\fR | |
362 | Creates a file named \fIname\fR relative to | |
363 | directory \fIdirectory\fR and write \fIcontents\fR | |
364 | to that file using the encoding [\fBencoding system\fR]. | |
365 | If \fIcontents\fR does not end with a newline, a newline | |
366 | will be appended so that the file named \fIname\fR | |
367 | does end with a newline. Because the system encoding is used, | |
368 | this command is only suitable for making text files. | |
369 | The file will be removed by the next evaluation | |
370 | of [\fBcleanupTests\fR], unless it is removed by | |
371 | [\fBremoveFile\fR] first. The default value of | |
372 | \fIdirectory\fR is the directory [\fBconfigure -tmpdir\fR]. | |
373 | Returns the full path of the file created. Use this command | |
374 | to create any text file required by a test with contents as needed. | |
375 | .TP | |
376 | \fBremoveFile\fR \fIname ?directory?\fR | |
377 | Forces the file referenced by \fIname\fR to be removed. This file name | |
378 | should be relative to \fIdirectory\fR. The default value of | |
379 | \fIdirectory\fR is the directory [\fBconfigure -tmpdir\fR]. | |
380 | Returns an empty string. Use this command to delete files | |
381 | created by [\fBmakeFile\fR]. | |
382 | .TP | |
383 | \fBmakeDirectory\fR \fIname ?directory?\fR | |
384 | Creates a directory named \fIname\fR relative to directory \fIdirectory\fR. | |
385 | The directory will be removed by the next evaluation of [\fBcleanupTests\fR], | |
386 | unless it is removed by [\fBremoveDirectory\fR] first. | |
387 | The default value of \fIdirectory\fR is the directory | |
388 | [\fBconfigure -tmpdir\fR]. | |
389 | Returns the full path of the directory created. Use this command | |
390 | to create any directories that are required to exist by a test. | |
391 | .TP | |
392 | \fBremoveDirectory\fR \fIname ?directory?\fR | |
393 | Forces the directory referenced by \fIname\fR to be removed. This | |
394 | directory should be relative to \fIdirectory\fR. | |
395 | The default value of \fIdirectory\fR is the directory | |
396 | [\fBconfigure -tmpdir\fR]. | |
397 | Returns an empty string. Use this command to delete any directories | |
398 | created by [\fBmakeDirectory\fR]. | |
399 | .TP | |
400 | \fBviewFile\fR \fIfile ?directory?\fR | |
401 | Returns the contents of \fIfile\fR, except for any | |
402 | final newline, just as [\fBread -nonewline\fR] would return. | |
403 | This file name should be relative to \fIdirectory\fR. | |
404 | The default value of \fIdirectory\fR is the directory | |
405 | [\fBconfigure -tmpdir\fR]. Use this command | |
406 | as a convenient way to turn the contents of a file generated | |
407 | by a test into the result of that test for matching against | |
408 | an expected result. The contents of the file are read using | |
409 | the system encoding, so its usefulness is limited to text | |
410 | files. | |
411 | .TP | |
412 | \fBcleanupTests\fR | |
413 | Intended to clean up and summarize after several tests have been | |
414 | run. Typically called once per test file, at the end of the file | |
415 | after all tests have been completed. For best effectiveness, be | |
416 | sure that the [\fBcleanupTests\fR] is evaluated even if an error | |
417 | occurs earlier in the test file evaluation. | |
418 | .sp | |
419 | Prints statistics about the tests run and removes files that were | |
420 | created by [\fBmakeDirectory\fR] and [\fBmakeFile\fR] since the | |
421 | last [\fBcleanupTests\fR]. Names of files and directories | |
422 | in the directory [\fBconfigure -tmpdir\fR] created since | |
423 | the last [\fBcleanupTests\fR], but not created by | |
424 | [\fBmakeFile\fR] or [\fBmakeDirectory\fR] are printed | |
425 | to [\fBoutputChannel\fR]. This command also restores the original | |
426 | shell environment, as described by the ::env | |
427 | array. Returns an empty string. | |
428 | .TP | |
429 | \fBrunAllTests\fR | |
430 | This is a master command meant to run an entire suite of tests, | |
431 | spanning multiple files and/or directories, as governed by | |
432 | the configurable options of \fBtcltest\fR. See \fBRUNNING ALL TESTS\fR | |
433 | below for a complete description of the many variations possible | |
434 | with [\fBrunAllTests\fR]. | |
435 | .SH "CONFIGURATION COMMANDS" | |
436 | .TP | |
437 | \fBconfigure\fR | |
438 | Returns the list of configurable options supported by \fBtcltest\fR. | |
439 | See \fBCONFIGURABLE OPTIONS\fR below for the full list of options, | |
440 | their valid values, and their effect on \fBtcltest\fR operations. | |
441 | .TP | |
442 | \fBconfigure \fIoption\fR | |
443 | Returns the current value of the supported configurable option \fIoption\fR. | |
444 | Raises an error if \fIoption\fR is not a supported configurable option. | |
445 | .TP | |
446 | \fBconfigure \fIoption value ?option value ...?\fR | |
447 | Sets the value of each configurable option \fIoption\fR to the | |
448 | corresponding value \fIvalue\fR, in order. Raises an error if | |
449 | an \fIoption\fR is not a supported configurable option, or if | |
450 | \fIvalue\fR is not a valid value for the corresponding \fIoption\fR, | |
451 | or if a \fIvalue\fR is not provided. When an error is raised, the | |
452 | operation of [\fBconfigure\fR] is halted, and subsequent \fIoption value\fR | |
453 | arguments are not processed. | |
454 | .sp | |
455 | If the environment variable \fB::env(TCLTEST_OPTIONS)\fR exists when | |
456 | the \fBtcltest\fR package is loaded (by [\fBpackage require tcltest\fR]) | |
457 | then its value is taken as a list of arguments to pass to [\fBconfigure\fR]. | |
458 | This allows the default values of the configuration options to be | |
459 | set by the environment. | |
460 | .TP | |
461 | \fBcustomMatch \fImode script\fR | |
462 | Registers \fImode\fR as a new legal value of the \fB-match\fR option | |
463 | to [\fBtest\fR]. When the \fB-match \fImode\fR option is | |
464 | passed to [\fBtest\fR], the script \fIscript\fR will be evaluated | |
465 | to compare the actual result of evaluating the body of the test | |
466 | to the expected result. | |
467 | To perform the match, the \fIscript\fR is completed with two additional | |
468 | words, the expected result, and the actual result, and the completed script | |
469 | is evaluated in the global namespace. | |
470 | The completed script is expected to return a boolean value indicating | |
471 | whether or not the results match. The built-in matching modes of | |
472 | [\fBtest\fR] are \fBexact\fR, \fBglob\fR, and \fBregexp\fR. | |
473 | .TP | |
474 | \fBtestConstraint \fIconstraint ?boolean?\fR | |
475 | Sets or returns the boolean value associated with the named \fIconstraint\fR. | |
476 | See \fBTEST CONSTRAINTS\fR below for more information. | |
477 | .TP | |
478 | \fBinterpreter\fR \fI?executableName?\fR | |
479 | Sets or returns the name of the executable to be [\fBexec\fR]ed by | |
480 | [\fBrunAllTests\fR] to run each test file when | |
481 | [\fBconfigure -singleproc\fR] is false. | |
482 | The default value for [\fBinterpreter\fR] is the name of the | |
483 | currently running program as returned by [\fBinfo nameofexecutable\fR]. | |
484 | .TP | |
485 | \fBoutputChannel\fR \fI?channelID?\fR | |
486 | Sets or returns the output channel ID. This defaults to stdout. | |
487 | Any test that prints test related output should send | |
488 | that output to [\fBoutputChannel\fR] rather than letting | |
489 | that output default to stdout. | |
490 | .TP | |
491 | \fBerrorChannel\fR \fI?channelID?\fR | |
492 | Sets or returns the error channel ID. This defaults to stderr. | |
493 | Any test that prints error messages should send | |
494 | that output to [\fBerrorChannel\fR] rather than printing | |
495 | directly to stderr. | |
496 | .SH "SHORTCUT COMMANDS" | |
497 | .TP | |
498 | \fBdebug \fI?level?\fR | |
499 | Same as [\fBconfigure -debug \fI?level?\fR]. | |
500 | .TP | |
501 | \fBerrorFile \fI?filename?\fR | |
502 | Same as [\fBconfigure -errfile \fI?filename?\fR]. | |
503 | .TP | |
504 | \fBlimitConstraints \fI?boolean?\fR | |
505 | Same as [\fBconfigure -limitconstraints \fI?boolean?\fR]. | |
506 | .TP | |
507 | \fBloadFile \fI?filename?\fR | |
508 | Same as [\fBconfigure -loadfile \fI?filename?\fR]. | |
509 | .TP | |
510 | \fBloadScript \fI?script?\fR | |
511 | Same as [\fBconfigure -load \fI?script?\fR]. | |
512 | .TP | |
513 | \fBmatch \fI?patternList?\fR | |
514 | Same as [\fBconfigure -match \fI?patternList?\fR]. | |
515 | .TP | |
516 | \fBmatchDirectories \fI?patternList?\fR | |
517 | Same as [\fBconfigure -relateddir \fI?patternList?\fR]. | |
518 | .TP | |
519 | \fBmatchFiles \fI?patternList?\fR | |
520 | Same as [\fBconfigure -file \fI?patternList?\fR]. | |
521 | .TP | |
522 | \fBoutputFile \fI?filename?\fR | |
523 | Same as [\fBconfigure -outfile \fI?filename?\fR]. | |
524 | .TP | |
525 | \fBpreserveCore \fI?level?\fR | |
526 | Same as [\fBconfigure -preservecore \fI?level?\fR]. | |
527 | .TP | |
528 | \fBsingleProcess \fI?boolean?\fR | |
529 | Same as [\fBconfigure -singleproc \fI?boolean?\fR]. | |
530 | .TP | |
531 | \fBskip \fI?patternList?\fR | |
532 | Same as [\fBconfigure -skip \fI?patternList?\fR]. | |
533 | .TP | |
534 | \fBskipDirectories \fI?patternList?\fR | |
535 | Same as [\fBconfigure -asidefromdir \fI?patternList?\fR]. | |
536 | .TP | |
537 | \fBskipFiles \fI?patternList?\fR | |
538 | Same as [\fBconfigure -notfile \fI?patternList?\fR]. | |
539 | .TP | |
540 | \fBtemporaryDirectory \fI?directory?\fR | |
541 | Same as [\fBconfigure -tmpdir \fI?directory?\fR]. | |
542 | .TP | |
543 | \fBtestsDirectory \fI?directory?\fR | |
544 | Same as [\fBconfigure -testdir \fI?directory?\fR]. | |
545 | .TP | |
546 | \fBverbose \fI?level?\fR | |
547 | Same as [\fBconfigure -verbose \fI?level?\fR]. | |
548 | .SH "OTHER COMMANDS" | |
549 | .PP | |
550 | The remaining commands provided by \fBtcltest\fR have better | |
551 | alternatives provided by \fBtcltest\fR or \fBTcl\fR itself. They | |
552 | are retained to support existing test suites, but should be avoided | |
553 | in new code. | |
554 | .TP | |
555 | \fBtest\fR \fIname description optionList\fR | |
556 | This form of [\fBtest\fR] was provided to enable passing many | |
557 | options spanning several lines to [\fBtest\fR] as a single | |
558 | argument quoted by braces, rather than needing to backslash quote | |
559 | the newlines between arguments to [\fBtest\fR]. The \fIoptionList\fR | |
560 | argument is expected to be a list with an even number of elements | |
561 | representing \fIoption\fR and \fIvalue\fR arguments to pass | |
562 | to [\fBtest\fR]. However, these values are not passed directly, as | |
563 | in the alternate forms of [\fBswitch\fR]. Instead, this form makes | |
564 | an unfortunate attempt to overthrow Tcl's substitution rules by | |
565 | performing substitutions on some of the list elements as an attempt to | |
566 | implement a ``do what I mean'' interpretation of a brace-enclosed | |
567 | ``block''. The result is nearly impossible to document clearly, and | |
568 | for that reason this form is not recommended. See the examples in | |
569 | \fBCREATING TEST SUITES WITH TCLTEST\fR below to see that this | |
570 | form is really not necessary to avoid backslash-quoted newlines. | |
571 | If you insist on using this form, examine | |
572 | the source code of \fBtcltest\fR if you want to know the substitution | |
573 | details, or just enclose the third through last argument | |
574 | to [\fBtest\fR] in braces and hope for the best. | |
575 | .TP | |
576 | \fBworkingDirectory\fR \fI?directoryName?\fR | |
577 | Sets or returns the current working directory when the test suite is | |
578 | running. The default value for workingDirectory is the directory in | |
579 | which the test suite was launched. The Tcl commands [\fBcd\fR] and | |
580 | [\fBpwd\fR] are sufficient replacements. | |
581 | .TP | |
582 | \fBnormalizeMsg\fR \fImsg\fR | |
583 | Returns the result of removing the ``extra'' newlines from \fImsg\fR, | |
584 | where ``extra'' is rather imprecise. Tcl offers plenty of string | |
585 | processing commands to modify strings as you wish, and | |
586 | [\fBcustomMatch\fR] allows flexible matching of actual and expected | |
587 | results. | |
588 | .TP | |
589 | \fBnormalizePath\fR \fIpathVar\fR | |
590 | Resolves symlinks in a path, thus creating a path without internal | |
591 | redirection. It is assumed that \fIpathVar\fR is absolute. | |
592 | \fIpathVar\fR is modified in place. The Tcl command [\fBfile normalize\fR] | |
593 | is a sufficient replacement. | |
594 | .TP | |
595 | \fBbytestring\fR \fIstring\fR | |
596 | Construct a string that consists of the requested sequence of bytes, | |
597 | as opposed to a string of properly formed UTF-8 characters using the | |
598 | value supplied in \fIstring\fR. This allows the tester to create | |
599 | denormalized or improperly formed strings to pass to C procedures that | |
600 | are supposed to accept strings with embedded NULL types and confirm | |
601 | that a string result has a certain pattern of bytes. This is | |
602 | exactly equivalent to the Tcl command [\fBencoding convertfrom identity\fR]. | |
603 | .SH TESTS | |
604 | .PP | |
605 | The [\fBtest\fR] command is the heart of the \fBtcltest\fR package. | |
606 | Its essential function is to evaluate a Tcl script and compare | |
607 | the result with an expected result. The options of [\fBtest\fR] | |
608 | define the test script, the environment in which to evaluate it, | |
609 | the expected result, and how the compare the actual result to | |
610 | the expected result. Some configuration options of \fBtcltest\fR | |
611 | also influence how [\fBtest\fR] operates. | |
612 | .PP | |
613 | The valid options for [\fBtest\fR] are summarized: | |
614 | .CS | |
615 | .ta 0.8i | |
616 | \fBtest\fR \fIname\fR \fIdescription\fR | |
617 | ?-constraints \fIkeywordList|expression\fR? | |
618 | ?-setup \fIsetupScript\fR? | |
619 | ?-body \fItestScript\fR? | |
620 | ?-cleanup \fIcleanupScript\fR? | |
621 | ?-result \fIexpectedAnswer\fR? | |
622 | ?-output \fIexpectedOutput\fR? | |
623 | ?-errorOutput \fIexpectedError\fR? | |
624 | ?-returnCodes \fIcodeList\fR? | |
625 | ?-match \fImode\fR? | |
626 | .CE | |
627 | The \fIname\fR may be any string. It is conventional to choose | |
628 | a \fIname\fR according to the pattern: | |
629 | .CS | |
630 | \fItarget\fR-\fImajorNum\fR.\fIminorNum\fR | |
631 | .CE | |
632 | For white-box (regression) tests, the target should be the name of the | |
633 | C function or Tcl procedure being tested. For black-box tests, the | |
634 | target should be the name of the feature being tested. Some conventions | |
635 | call for the names of black-box tests to have the suffix \fB_bb\fR. | |
636 | Related tests should share a major number. As a test suite evolves, | |
637 | it is best to have the same test name continue to correspond to the | |
638 | same test, so that it remains meaningful to say things like ``Test | |
639 | foo-1.3 passed in all releases up to 3.4, but began failing in | |
640 | release 3.5.'' | |
641 | .PP | |
642 | During evaluation of [\fBtest\fR], the \fIname\fR will be compared | |
643 | to the lists of string matching patterns returned by | |
644 | [\fBconfigure -match\fR], and [\fBconfigure -skip\fR]. The test | |
645 | will be run only if \fIname\fR matches any of the patterns from | |
646 | [\fBconfigure -match\fR] and matches none of the patterns | |
647 | from [\fBconfigure -skip\fR]. | |
648 | .PP | |
649 | The \fIdescription\fR should be a short textual description of the | |
650 | test. The \fIdescription\fR is included in output produced by the | |
651 | test, typically test failure messages. Good \fIdescription\fR values | |
652 | should briefly explain the purpose of the test to users of a test suite. | |
653 | The name of a Tcl or C function being tested should be included in the | |
654 | description for regression tests. If the test case exists to reproduce | |
655 | a bug, include the bug ID in the description. | |
656 | .PP | |
657 | Valid attributes and associated values are: | |
658 | .TP | |
659 | \fB-constraints \fIkeywordList|expression\fR | |
660 | The optional \fB-constraints\fR attribute can be list of one or more | |
661 | keywords or an expression. If the \fB-constraints\fR value is a list of | |
662 | keywords, each of these keywords should be the name of a constraint | |
663 | defined by a call to [\fBtestConstraint\fR]. If any of the listed | |
664 | constraints is false or does not exist, the test is skipped. If the | |
665 | \fB-constraints\fR value is an expression, that expression | |
666 | is evaluated. If the expression evaluates to true, then the test is run. | |
667 | Note that the expression form of \fB-constraints\fR may interfere with the | |
668 | operation of [\fBconfigure -constraints\fR] and | |
669 | [\fBconfigure -limitconstraints\fR], and is not recommended. | |
670 | Appropriate constraints should be added to any tests that should | |
671 | not always be run. That is, conditional evaluation of a test | |
672 | should be accomplished by the \fB-constraints\fR option, not by | |
673 | conditional evaluation of [\fBtest\fR]. In that way, the same | |
674 | number of tests are always reported by the test suite, though | |
675 | the number skipped may change based on the testing environment. | |
676 | The default value is an empty list. | |
677 | See \fBTEST CONSTRAINTS\fR below for a list of built-in constraints | |
678 | and information on how to add your own constraints. | |
679 | .TP | |
680 | \fB-setup \fIscript\fR | |
681 | The optional \fB-setup\fR attribute indicates a \fIscript\fR that will be run | |
682 | before the script indicated by the \fB-body\fR attribute. If evaluation | |
683 | of \fIscript\fR raises an error, the test will fail. The default value | |
684 | is an empty script. | |
685 | .TP | |
686 | \fB-body \fIscript\fR | |
687 | The \fB-body\fR attribute indicates the \fIscript\fR to run to carry out the | |
688 | test. It must return a result that can be checked for correctness. | |
689 | If evaluation of \fIscript\fR raises an error, the test will fail. | |
690 | The default value is an empty script. | |
691 | .TP | |
692 | \fB-cleanup \fIscript\fR | |
693 | The optional \fB-cleanup\fR attribute indicates a \fIscript\fR that will be | |
694 | run after the script indicated by the \fB-body\fR attribute. | |
695 | If evaluation of \fIscript\fR raises an error, the test will fail. | |
696 | The default value is an empty script. | |
697 | .TP | |
698 | \fB-match \fImode\fR | |
699 | The \fB-match\fR attribute determines how expected answers supplied by | |
700 | \fB-result\fR, \fB-output\fR, and \fB-errorOutput\fR are compared. Valid | |
701 | values for \fImode\fR are \fBregexp\fR, \fBglob\fR, \fBexact\fR, and | |
702 | any value registered by a prior call to [\fBcustomMatch\fR]. The default | |
703 | value is \fBexact\fR. | |
704 | .TP | |
705 | \fB-result \fIexpectedValue\fR | |
706 | The \fB-result\fR attribute supplies the \fIexpectedValue\fR against which | |
707 | the return value from script will be compared. The default value is | |
708 | an empty string. | |
709 | .TP | |
710 | \fB-output \fIexpectedValue\fR | |
711 | The \fB-output\fR attribute supplies the \fIexpectedValue\fR against which | |
712 | any output sent to \fBstdout\fR or [\fBoutputChannel\fR] during evaluation | |
713 | of the script(s) will be compared. Note that only output printed using | |
714 | [\fB::puts\fR] is used for comparison. If \fB-output\fR is not specified, | |
715 | output sent to \fBstdout\fR and [\fBoutputChannel\fR] is not processed for | |
716 | comparison. | |
717 | .TP | |
718 | \fB-errorOutput \fIexpectedValue\fR | |
719 | The \fB-errorOutput\fR attribute supplies the \fIexpectedValue\fR against | |
720 | which any output sent to \fBstderr\fR or [\fBerrorChannel\fR] during | |
721 | evaluation of the script(s) will be compared. Note that only output | |
722 | printed using [\fB::puts\fR] is used for comparison. If \fB-errorOutput\fR | |
723 | is not specified, output sent to \fBstderr\fR and [\fBerrorChannel\fR] is | |
724 | not processed for comparison. | |
725 | .TP | |
726 | \fB-returnCodes \fIexpectedCodeList\fR | |
727 | The optional \fB-returnCodes\fR attribute supplies \fIexpectedCodeList\fR, | |
728 | a list of return codes that may be accepted from evaluation of the | |
729 | \fB-body\fR script. If evaluation of the \fB-body\fR script returns | |
730 | a code not in the \fIexpectedCodeList\fR, the test fails. All | |
731 | return codes known to [\fBreturn\fR], in both numeric and symbolic | |
732 | form, including extended return codes, are acceptable elements in | |
733 | the \fIexpectedCodeList\fR. Default value is \fB{ok return}\fR. | |
734 | .PP | |
735 | To pass, a test must successfully evaluate its \fB-setup\fR, \fB-body\fR, | |
736 | and \fB-cleanup\fR scripts. The return code of the \fB-body\fR script and | |
737 | its result must match expected values, and if specified, output and error | |
738 | data from the test must match expected \fB-output\fR and \fB-errorOutput\fR | |
739 | values. If any of these conditions are not met, then the test fails. | |
740 | Note that all scripts are evaluated in the context of the caller | |
741 | of [\fBtest\fR]. | |
742 | .PP | |
743 | As long as [\fBtest\fR] is called with valid syntax and legal | |
744 | values for all attributes, it will not raise an error. Test | |
745 | failures are instead reported as output written to [\fBoutputChannel\fR]. | |
746 | In default operation, a successful test produces no output. The output | |
747 | messages produced by [\fBtest\fR] are controlled by the | |
748 | [\fBconfigure -verbose\fR] option as described in \fBCONFIGURABLE OPTIONS\fR | |
749 | below. Any output produced by the test scripts themselves should be | |
750 | produced using [\fB::puts\fR] to [\fBoutputChannel\fR] or | |
751 | [\fBerrorChannel\fR], so that users of the test suite may | |
752 | easily capture output with the [\fBconfigure -outfile\fR] and | |
753 | [\fBconfigure -errfile\fR] options, and so that the \fB-output\fR | |
754 | and \fB-errorOutput\fR attributes work properly. | |
755 | .SH "TEST CONSTRAINTS" | |
756 | .PP | |
757 | Constraints are used to determine whether or not a test should be skipped. | |
758 | Each constraint has a name, which may be any string, and a boolean | |
759 | value. Each [\fBtest\fR] has a \fB-constraints\fR value which is a | |
760 | list of constraint names. There are two modes of constraint control. | |
761 | Most frequently, the default mode is used, indicated by a setting | |
762 | of [\fBconfigure -limitconstraints\fR] to false. The test will run | |
763 | only if all constraints in the list are true-valued. Thus, | |
764 | the \fB-constraints\fR option of [\fBtest\fR] is a convenient, symbolic | |
765 | way to define any conditions required for the test to be possible or | |
766 | meaningful. For example, a [\fBtest\fR] with \fB-constraints unix\fR | |
767 | will only be run if the constraint \fBunix\fR is true, which indicates | |
768 | the test suite is being run on a Unix platform. | |
769 | .PP | |
770 | Each [\fBtest\fR] should include whatever \fB-constraints\fR are | |
771 | required to constrain it to run only where appropriate. Several | |
772 | constraints are pre-defined in the \fBtcltest\fR package, listed | |
773 | below. The registration of user-defined constraints is performed | |
774 | by the [\fBtestConstraint\fR] command. User-defined constraints | |
775 | may appear within a test file, or within the script specified | |
776 | by the [\fBconfigure -load\fR] or [\fBconfigure -loadfile\fR] | |
777 | options. | |
778 | .PP | |
779 | The following is a list of constraints pre-defined by the | |
780 | \fBtcltest\fR package itself: | |
781 | .TP | |
782 | \fIsingleTestInterp\fR | |
783 | test can only be run if all test files are sourced into a single interpreter | |
784 | .TP | |
785 | \fIunix\fR | |
786 | test can only be run on any Unix platform | |
787 | .TP | |
788 | \fIwin\fR | |
789 | test can only be run on any Windows platform | |
790 | .TP | |
791 | \fInt\fR | |
792 | test can only be run on any Windows NT platform | |
793 | .TP | |
794 | \fI95\fR | |
795 | test can only be run on any Windows 95 platform | |
796 | .TP | |
797 | \fI98\fR | |
798 | test can only be run on any Windows 98 platform | |
799 | .TP | |
800 | \fImac\fR | |
801 | test can only be run on any Mac platform | |
802 | .TP | |
803 | \fIunixOrWin\fR | |
804 | test can only be run on a Unix or Windows platform | |
805 | .TP | |
806 | \fImacOrWin\fR | |
807 | test can only be run on a Mac or Windows platform | |
808 | .TP | |
809 | \fImacOrUnix\fR | |
810 | test can only be run on a Mac or Unix platform | |
811 | .TP | |
812 | \fItempNotWin\fR | |
813 | test can not be run on Windows. This flag is used to temporarily | |
814 | disable a test. | |
815 | .TP | |
816 | \fItempNotMac\fR | |
817 | test can not be run on a Mac. This flag is used | |
818 | to temporarily disable a test. | |
819 | .TP | |
820 | \fIunixCrash\fR | |
821 | test crashes if it's run on Unix. This flag is used to temporarily | |
822 | disable a test. | |
823 | .TP | |
824 | \fIwinCrash\fR | |
825 | test crashes if it's run on Windows. This flag is used to temporarily | |
826 | disable a test. | |
827 | .TP | |
828 | \fImacCrash\fR | |
829 | test crashes if it's run on a Mac. This flag is used to temporarily | |
830 | disable a test. | |
831 | .TP | |
832 | \fIemptyTest\fR | |
833 | test is empty, and so not worth running, but it remains as a | |
834 | place-holder for a test to be written in the future. This constraint | |
835 | has value false to cause tests to be skipped unless the user specifies | |
836 | otherwise. | |
837 | .TP | |
838 | \fIknownBug\fR | |
839 | test is known to fail and the bug is not yet fixed. This constraint | |
840 | has value false to cause tests to be skipped unless the user specifies | |
841 | otherwise. | |
842 | .TP | |
843 | \fInonPortable\fR | |
844 | test can only be run in some known development environment. | |
845 | Some tests are inherently non-portable because they depend on things | |
846 | like word length, file system configuration, window manager, etc. | |
847 | This constraint has value false to cause tests to be skipped unless | |
848 | the user specifies otherwise. | |
849 | .TP | |
850 | \fIuserInteraction\fR | |
851 | test requires interaction from the user. This constraint has | |
852 | value false to causes tests to be skipped unless the user specifies | |
853 | otherwise. | |
854 | .TP | |
855 | \fIinteractive\fR | |
856 | test can only be run in if the interpreter is in interactive mode | |
857 | (when the global tcl_interactive variable is set to 1). | |
858 | .TP | |
859 | \fInonBlockFiles\fR | |
860 | test can only be run if platform supports setting files into | |
861 | nonblocking mode | |
862 | .TP | |
863 | \fIasyncPipeClose\fR | |
864 | test can only be run if platform supports async flush and async close | |
865 | on a pipe | |
866 | .TP | |
867 | \fIunixExecs\fR | |
868 | test can only be run if this machine has Unix-style commands | |
869 | \fBcat\fR, \fBecho\fR, \fBsh\fR, \fBwc\fR, \fBrm\fR, \fBsleep\fR, | |
870 | \fBfgrep\fR, \fBps\fR, \fBchmod\fR, and \fBmkdir\fR available | |
871 | .TP | |
872 | \fIhasIsoLocale\fR | |
873 | test can only be run if can switch to an ISO locale | |
874 | .TP | |
875 | \fIroot\fR | |
876 | test can only run if Unix user is root | |
877 | .TP | |
878 | \fInotRoot\fR | |
879 | test can only run if Unix user is not root | |
880 | .TP | |
881 | \fIeformat\fR | |
882 | test can only run if app has a working version of sprintf with respect | |
883 | to the "e" format of floating-point numbers. | |
884 | .TP | |
885 | \fIstdio\fR | |
886 | test can only be run if [\fBinterpreter\fR] can be [\fBopen\fR]ed | |
887 | as a pipe. | |
888 | .PP | |
889 | The alternative mode of constraint control is enabled by setting | |
890 | [\fBconfigure -limitconstraints\fR] to true. With that configuration | |
891 | setting, all existing constraints other than those in the constraint | |
892 | list returned by [\fBconfigure -constraints\fR] are set to false. | |
893 | When the value of [\fBconfigure -constraints\fR] | |
894 | is set, all those constraints are set to true. The effect is that | |
895 | when both options [\fBconfigure -constraints\fR] and | |
896 | [\fBconfigure -limitconstraints\fR] are in use, only those tests including | |
897 | only constraints from the [\fBconfigure -constraints\fR] list | |
898 | are run; all others are skipped. For example, one might set | |
899 | up a configuration with | |
900 | .CS | |
901 | \fBconfigure\fR -constraints knownBug \e | |
902 | -limitconstraints true \e | |
903 | -verbose pass | |
904 | .CE | |
905 | to run exactly those tests that exercise known bugs, and discover | |
906 | whether any of them pass, indicating the bug had been fixed. | |
907 | .SH "RUNNING ALL TESTS" | |
908 | .PP | |
909 | The single command [\fBrunAllTests\fR] is evaluated to run an entire | |
910 | test suite, spanning many files and directories. The configuration | |
911 | options of \fBtcltest\fR control the precise operations. The | |
912 | [\fBrunAllTests\fR] command begins by printing a summary of its | |
913 | configuration to [\fBoutputChannel\fR]. | |
914 | .PP | |
915 | Test files to be evaluated are sought in the directory | |
916 | [\fBconfigure -testdir\fR]. The list of files in that directory | |
917 | that match any of the patterns in [\fBconfigure -file\fR] and | |
918 | match none of the patterns in [\fBconfigure -notfile\fR] is generated | |
919 | and sorted. Then each file will be evaluated in turn. If | |
920 | [\fBconfigure -singleproc\fR] is true, then each file will | |
921 | be [\fBsource\fR]d in the caller's context. If it is false, | |
922 | then a copy of [\fBinterpreter\fR] will be [\fBexec\fR]d to | |
923 | evaluate each file. The multi-process operation is useful | |
924 | when testing can cause errors so severe that a process | |
925 | terminates. Although such an error may terminate a child | |
926 | process evaluating one file, the master process can continue | |
927 | with the rest of the test suite. In multi-process operation, | |
928 | the configuration of \fBtcltest\fR in the master process is | |
929 | passed to the child processes as command line arguments, | |
930 | with the exception of [\fBconfigure -outfile\fR]. The | |
931 | [\fBrunAllTests\fR] command in the | |
932 | master process collects all output from the child processes | |
933 | and collates their results into one master report. Any | |
934 | reports of individual test failures, or messages requested | |
935 | by a [\fBconfigure -verbose\fR] setting are passed directly | |
936 | on to [\fBoutputChannel\fR] by the master process. | |
937 | .PP | |
938 | After evaluating all selected test files, a summary of the | |
939 | results is printed to [\fBoutputChannel\fR]. The summary | |
940 | includes the total number of [\fBtest\fR]s evaluated, broken | |
941 | down into those skipped, those passed, and those failed. | |
942 | The summary also notes the number of files evaluated, and the names | |
943 | of any files with failing tests or errors. A list of | |
944 | the constraints that caused tests to be skipped, and the | |
945 | number of tests skipped for each is also printed. Also, | |
946 | messages are printed if it appears that evaluation of | |
947 | a test file has caused any temporary files to be left | |
948 | behind in [\fBconfigure -tmpdir\fR]. | |
949 | .PP | |
950 | Having completed and summarized all selected test files, | |
951 | [\fBrunAllTests\fR] then recursively acts on subdirectories | |
952 | of [\fBconfigure -testdir\fR]. All subdirectories that | |
953 | match any of the patterns in [\fBconfigure -relateddir\fR] | |
954 | and do not match any of the patterns in | |
955 | [\fBconfigure -asidefromdir\fR] are examined. If | |
956 | a file named \fBall.tcl\fR is found in such a directory, | |
957 | it will be [\fBsource\fR]d in the caller's context. | |
958 | Whether or not an examined directory contains an | |
959 | \fBall.tcl\fR file, its subdirectories are also scanned | |
960 | against the [\fBconfigure -relateddir\fR] and | |
961 | [\fBconfigure -asidefromdir\fR] patterns. In this way, | |
962 | many directories in a directory tree can have all their | |
963 | test files evaluated by a single [\fBrunAllTests\fR] | |
964 | command. | |
965 | .SH "CONFIGURABLE OPTIONS" | |
966 | The [\fBconfigure\fR] command is used to set and query the configurable | |
967 | options of \fBtcltest\fR. The valid options are: | |
968 | .TP | |
969 | \fB-singleproc \fIboolean\fR | |
970 | Controls whether or not [\fBrunAllTests\fR] spawns a child process for | |
971 | each test file. No spawning when \fIboolean\fR is true. Default | |
972 | value is false. | |
973 | .TP | |
974 | \fB-debug \fIlevel\fR | |
975 | Sets the debug level to \fIlevel\fR, an integer value indicating how | |
976 | much debugging information should be printed to stdout. Note that | |
977 | debug messages always go to stdout, independent of the value of | |
978 | [\fBconfigure -outfile\fR]. Default value is 0. Levels are defined as: | |
979 | .RS | |
980 | .IP 0 | |
981 | Do not display any debug information. | |
982 | .IP 1 | |
983 | Display information regarding whether a test is skipped because it | |
984 | doesn't match any of the tests that were specified using by | |
985 | [\fBconfigure -match\fR] (userSpecifiedNonMatch) or matches any of | |
986 | the tests specified by [\fBconfigure -skip\fR] (userSpecifiedSkip). Also | |
987 | print warnings about possible lack of cleanup or balance in test files. | |
988 | Also print warnings about any re-use of test names. | |
989 | .IP 2 | |
990 | Display the flag array parsed by the command line processor, the | |
991 | contents of the ::env array, and all user-defined variables that exist | |
992 | in the current namespace as they are used. | |
993 | .IP 3 | |
994 | Display information regarding what individual procs in the test | |
995 | harness are doing. | |
996 | .RE | |
997 | .TP | |
998 | \fB-verbose \fIlevel\fR | |
999 | Sets the type of output verbosity desired to \fIlevel\fR, | |
1000 | a list of zero or more of the elements \fBbody\fR, \fBpass\fR, | |
1001 | \fBskip\fR, \fBstart\fR, and \fBerror\fR. Default value | |
1002 | is \fB{body error}\fR. | |
1003 | Levels are defined as: | |
1004 | .RS | |
1005 | .IP "body (b)" | |
1006 | Display the body of failed tests | |
1007 | .IP "pass (p)" | |
1008 | Print output when a test passes | |
1009 | .IP "skip (s)" | |
1010 | Print output when a test is skipped | |
1011 | .IP "start (t)" | |
1012 | Print output whenever a test starts | |
1013 | .IP "error (e)" | |
1014 | Print errorInfo and errorCode, if they exist, when a test return code | |
1015 | does not match its expected return code | |
1016 | .RE | |
1017 | The single letter abbreviations noted above are also recognized | |
1018 | so that [\fBconfigure -verbose pt\fR] is the same as | |
1019 | [\fBconfigure -verbose {pass start}\fR]. | |
1020 | .TP | |
1021 | \fB-preservecore \fIlevel\fR | |
1022 | Sets the core preservation level to \fIlevel\fR. This level | |
1023 | determines how stringent checks for core files are. Default | |
1024 | value is 0. Levels are defined as: | |
1025 | .RS | |
1026 | .IP 0 | |
1027 | No checking - do not check for core files at the end of each test | |
1028 | command, but do check for them in [\fBrunAllTests\fR] after all | |
1029 | test files have been evaluated. | |
1030 | .IP 1 | |
1031 | Also check for core files at the end of each [\fBtest\fR] command. | |
1032 | .IP 2 | |
1033 | Check for core files at all times described above, and save a | |
1034 | copy of each core file produced in [\fBconfigure -tmpdir\fR]. | |
1035 | .RE | |
1036 | .TP | |
1037 | \fB-limitconstraints \fIboolean\fR | |
1038 | Sets the mode by which [\fBtest\fR] honors constraints as described | |
1039 | in \fBTESTS\fR above. Default value is false. | |
1040 | .TP | |
1041 | \fB-constraints \fIlist\fR | |
1042 | Sets all the constraints in \fIlist\fR to true. Also used in | |
1043 | combination with [\fBconfigure -limitconstraints true\fR] to control an | |
1044 | alternative constraint mode as described in \fBTESTS\fR above. | |
1045 | Default value is an empty list. | |
1046 | .TP | |
1047 | \fB-tmpdir \fIdirectory\fR | |
1048 | Sets the temporary directory to be used by [\fBmakeFile\fR], | |
1049 | [\fBmakeDirectory\fR], [\fBviewFile\fR], [\fBremoveFile\fR], | |
1050 | and [\fBremoveDirectory\fR] as the default directory where | |
1051 | temporary files and directories created by test files should | |
1052 | be created. Default value is [\fBworkingDirectory\fR]. | |
1053 | .TP | |
1054 | \fB-testdir \fIdirectory\fR | |
1055 | Sets the directory searched by [\fBrunAllTests\fR] for test files | |
1056 | and subdirectories. Default value is [\fBworkingDirectory\fR]. | |
1057 | .TP | |
1058 | \fB-file \fIpatternList\fR | |
1059 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine | |
1060 | what test files to evaluate. Default value is \fB*.test\fR. | |
1061 | .TP | |
1062 | \fB-notfile \fIpatternList\fR | |
1063 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine | |
1064 | what test files to skip. Default value is \fBl.*.test\fR, so | |
1065 | that any SCCS lock files are skipped. | |
1066 | .TP | |
1067 | \fB-relateddir \fIpatternList\fR | |
1068 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine | |
1069 | what subdirectories to search for an \fBall.tcl\fR file. Default | |
1070 | value is \fB*\fR. | |
1071 | .TP | |
1072 | \fB-asidefromdir \fIpatternList\fR | |
1073 | Sets the list of patterns used by [\fBrunAllTests\fR] to determine | |
1074 | what subdirectories to skip when searching for an \fBall.tcl\fR file. | |
1075 | Default value is an empty list. | |
1076 | .TP | |
1077 | \fB-match \fIpatternList\fR | |
1078 | Set the list of patterns used by [\fBtest\fR] to determine whether | |
1079 | a test should be run. Default value is \fB*\fR. | |
1080 | .TP | |
1081 | \fB-skip \fIpatternList\fR | |
1082 | Set the list of patterns used by [\fBtest\fR] to determine whether | |
1083 | a test should be skipped. Default value is an empty list. | |
1084 | .TP | |
1085 | \fB-load \fIscript\fR | |
1086 | Sets a script to be evaluated by [\fBloadTestedCommands\fR]. | |
1087 | Default value is an empty script. | |
1088 | .TP | |
1089 | \fB-loadfile \fIfilename\fR | |
1090 | Sets the filename from which to read a script to be evaluated | |
1091 | by [\fBloadTestedCommands\fR]. This is an alternative to | |
1092 | \fB-load\fR. They cannot be used together. | |
1093 | .TP | |
1094 | \fB-outfile \fIfilename\fR | |
1095 | Sets the file to which all output produced by tcltest should be | |
1096 | written. A file named \fIfilename\fR will be [\fBopen\fR]ed for writing, | |
1097 | and the resulting channel will be set as the value of [\fBoutputChannel\fR]. | |
1098 | .TP | |
1099 | \fB-errfile \fIfilename\fR | |
1100 | Sets the file to which all error output produced by tcltest | |
1101 | should be written. A file named \fIfilename\fR will be [\fBopen\fR]ed | |
1102 | for writing, and the resulting channel will be set as the value | |
1103 | of [\fBerrorChannel\fR]. | |
1104 | .SH "CREATING TEST SUITES WITH TCLTEST" | |
1105 | .PP | |
1106 | The fundamental element of a test suite is the individual [\fBtest\fR] | |
1107 | command. We begin with several examples. | |
1108 | .IP [1] | |
1109 | Test of a script that returns normally. | |
1110 | .CS | |
1111 | \fBtest\fR example-1.0 {normal return} { | |
1112 | format %s value | |
1113 | } value | |
1114 | .CE | |
1115 | .IP [2] | |
1116 | Test of a script that requires context setup and cleanup. Note the | |
1117 | bracing and indenting style that avoids any need for line continuation. | |
1118 | .CS | |
1119 | \fBtest\fR example-1.1 {test file existence} -setup { | |
1120 | set file [makeFile {} test] | |
1121 | } -body { | |
1122 | file exists $file | |
1123 | } -cleanup { | |
1124 | removeFile test | |
1125 | } -result 1 | |
1126 | .CE | |
1127 | .IP [3] | |
1128 | Test of a script that raises an error. | |
1129 | .CS | |
1130 | \fBtest\fR example-1.2 {error return} -body { | |
1131 | error message | |
1132 | } -returnCodes error -result message | |
1133 | .CE | |
1134 | .IP [4] | |
1135 | Test with a constraint. | |
1136 | .CS | |
1137 | \fBtest\fR example-1.3 {user owns created files} -constraints { | |
1138 | unix | |
1139 | } -setup { | |
1140 | set file [makeFile {} test] | |
1141 | } -body { | |
1142 | file attributes $file -owner | |
1143 | } -cleanup { | |
1144 | removeFile test | |
1145 | } -result $::tcl_platform(user) | |
1146 | .CE | |
1147 | .PP | |
1148 | At the next higher layer of organization, several [\fBtest\fR] commands | |
1149 | are gathered together into a single test file. Test files should have | |
1150 | names with the \fB.test\fR extension, because that is the default pattern | |
1151 | used by [\fBrunAllTests\fR] to find test files. It is a good rule of | |
1152 | thumb to have one test file for each source code file of your project. | |
1153 | It is good practice to edit the test file and the source code file | |
1154 | together, keeping tests synchronized with code changes. | |
1155 | .PP | |
1156 | Most of the code in the test file should be the [\fBtest\fR] commands. | |
1157 | Use constraints to skip tests, rather than conditional evaluation | |
1158 | of [\fBtest\fR]. That is, do this: | |
1159 | .IP [5] | |
1160 | .CS | |
1161 | \fBtestConstraint\fR X [expr $myRequirement] | |
1162 | \fBtest\fR goodConditionalTest {} X { | |
1163 | # body | |
1164 | } result | |
1165 | .CE | |
1166 | and do not do this: | |
1167 | .IP [6] | |
1168 | .CS | |
1169 | if $myRequirement { | |
1170 | test badConditionalTest {} { | |
1171 | #body | |
1172 | } result | |
1173 | } | |
1174 | .CE | |
1175 | .PP | |
1176 | Use the \fB-setup\fR and \fB-cleanup\fR options to establish and release | |
1177 | all context requirements of the test body. Do not make tests depend on | |
1178 | prior tests in the file. Those prior tests might be skipped. If several | |
1179 | consecutive tests require the same context, the appropriate setup | |
1180 | and cleanup scripts may be stored in variable for passing to each tests | |
1181 | \fB-setup\fR and \fB-cleanup\fR options. This is a better solution than | |
1182 | performing setup outside of [\fBtest\fR] commands, because the setup will | |
1183 | only be done if necessary, and any errors during setup will be reported, | |
1184 | and not cause the test file to abort. | |
1185 | .PP | |
1186 | A test file should be able to be combined with other test files and not | |
1187 | interfere with them, even when [\fBconfigure -singleproc 1\fR] causes | |
1188 | all files to be evaluated in a common interpreter. A simple way to | |
1189 | achieve this is to have your tests define all their commands and variables | |
1190 | in a namespace that is deleted when the test file evaluation is complete. | |
1191 | A good namespace to use is a child namespace \fBtest\fR of the namespace | |
1192 | of the module you are testing. | |
1193 | .PP | |
1194 | A test file should also be able to be evaluated directly as a script, | |
1195 | not depending on being called by a master [\fBrunAllTests\fR]. This | |
1196 | means that each test file should process command line arguments to give | |
1197 | the tester all the configuration control that \fBtcltest\fR provides. | |
1198 | .PP | |
1199 | After all [\fBtest\fR]s in a test file, the command [\fBcleanupTests\fR] | |
1200 | should be called. | |
1201 | .IP [7] | |
1202 | Here is a sketch of a sample test file illustrating those points: | |
1203 | .CS | |
1204 | package require tcltest 2.2 | |
1205 | eval \fB::tcltest::configure\fR $argv | |
1206 | package require example | |
1207 | namespace eval ::example::test { | |
1208 | namespace import ::tcltest::* | |
1209 | \fBtestConstraint\fR X [expr {...}] | |
1210 | variable SETUP {#common setup code} | |
1211 | variable CLEANUP {#common cleanup code} | |
1212 | \fBtest\fR example-1 {} -setup $SETUP -body { | |
1213 | # First test | |
1214 | } -cleanup $CLEANUP -result {...} | |
1215 | \fBtest\fR example-2 {} -constraints X -setup $SETUP -body { | |
1216 | # Second test; constrained | |
1217 | } -cleanup $CLEANUP -result {...} | |
1218 | \fBtest\fR example-3 {} { | |
1219 | # Third test; no context required | |
1220 | } {...} | |
1221 | \fBcleanupTests\fR | |
1222 | } | |
1223 | namespace delete ::example::test | |
1224 | .CE | |
1225 | .PP | |
1226 | The next level of organization is a full test suite, made up of several | |
1227 | test files. One script is used to control the entire suite. The | |
1228 | basic function of this script is to call [\fBrunAllTests\fR] after | |
1229 | doing any necessary setup. This script is usually named \fBall.tcl\fR | |
1230 | because that's the default name used by [\fBrunAllTests\fR] when combining | |
1231 | multiple test suites into one testing run. | |
1232 | .IP [8] | |
1233 | Here is a sketch of a sample test suite master script: | |
1234 | .CS | |
1235 | package require Tcl 8.4 | |
1236 | package require tcltest 2.2 | |
1237 | package require example | |
1238 | \fB::tcltest::configure\fR -testdir \ | |
1239 | [file dirname [file normalize [info script]]] | |
1240 | eval \fB::tcltest::configure\fR $argv | |
1241 | \fB::tcltest::runAllTests\fR | |
1242 | .CE | |
1243 | .SH COMPATIBILITY | |
1244 | .PP | |
1245 | A number of commands and variables in the \fB::tcltest\fR namespace | |
1246 | provided by earlier releases of \fBtcltest\fR have not been documented | |
1247 | here. They are no longer part of the supported public interface of | |
1248 | \fBtcltest\fR and should not be used in new test suites. However, | |
1249 | to continue to support existing test suites written to the older | |
1250 | interface specifications, many of those deprecated commands and | |
1251 | variables still work as before. For example, in many circumstances, | |
1252 | [\fBconfigure\fR] will be automatically called shortly after | |
1253 | [\fBpackage require tcltest 2.1\fR] succeeds with arguments | |
1254 | from the variable \fB::argv\fR. This is to support test suites | |
1255 | that depend on the old behavior that \fBtcltest\fR was automatically | |
1256 | configured from command line arguments. New test files should not | |
1257 | depend on this, but should explicitly include | |
1258 | .CS | |
1259 | eval \fB::tcltest::configure\fR $::argv | |
1260 | .CE | |
1261 | to establish a configuration from command line arguments. | |
1262 | .SH "KNOWN ISSUES" | |
1263 | There are two known issues related to nested evaluations of [\fBtest\fR]. | |
1264 | The first issue relates to the stack level in which test scripts are | |
1265 | executed. Tests nested within other tests may be executed at the same | |
1266 | stack level as the outermost test. For example, in the following code: | |
1267 | .CS | |
1268 | \fBtest\fR level-1.1 {level 1} { | |
1269 | -body { | |
1270 | \fBtest\fR level-2.1 {level 2} { | |
1271 | } | |
1272 | } | |
1273 | } | |
1274 | .CE | |
1275 | any script executed in level-2.1 may be executed at the same stack | |
1276 | level as the script defined for level-1.1. | |
1277 | .PP | |
1278 | In addition, while two [\fBtest\fR]s have been run, results will only | |
1279 | be reported by [\fBcleanupTests\fR] for tests at the same level as | |
1280 | test level-1.1. However, test results for all tests run prior to | |
1281 | level-1.1 will be available when test level-2.1 runs. What this | |
1282 | means is that if you try to access the test results for test level-2.1, | |
1283 | it will may say that 'm' tests have run, 'n' tests have | |
1284 | been skipped, 'o' tests have passed and 'p' tests have failed, | |
1285 | where 'm', 'n', 'o', and 'p' refer to tests that were run at the | |
1286 | same test level as test level-1.1. | |
1287 | .PP | |
1288 | Implementation of output and error comparison in the test command | |
1289 | depends on usage of ::puts in your application code. Output is | |
1290 | intercepted by redefining the ::puts command while the defined test | |
1291 | script is being run. Errors thrown by C procedures or printed | |
1292 | directly from C applications will not be caught by the test command. | |
1293 | Therefore, usage of the \fB-output\fR and \fB-errorOuput\fR | |
1294 | options to [\fBtest\fR] is useful only for pure Tcl applications | |
1295 | that use [\fB::puts\fR] to produce output. | |
1296 | ||
1297 | .SH KEYWORDS | |
1298 | test, test harness, test suite |