Package cssutils :: Package tests :: Module test_tokenize2
[hide private]
[frames] | no frames]

Source Code for Module cssutils.tests.test_tokenize2

  1  # -*- coding: utf-8 -*- 
  2  """testcases for new cssutils.tokenize.Tokenizer 
  3   
  4  TODO:: 
  5   
  6      - escape ends with explicit space but \r\n as single space 
  7      - ur'"\""': [('STRING', ur'"\""', 1, 1)], 
  8      - font-face with escaped "-" 
  9   
 10  + old tests as new ones are **not complete**! 
 11  """ 
 12  __author__ = '$LastChangedBy: cthedot $' 
 13  __date__ = '$LastChangedDate: 2007-09-01 15:56:36 +0200 (Sa, 01 Sep 2007) $' 
 14  __version__ = '$LastChangedRevision: 302 $' 
 15   
 16  import xml.dom 
 17  import basetest 
 18  from cssutils.tokenize2 import * 
 19   
20 -class TokenizerTestCase(basetest.BaseTestCase):
21 22 testsall = { 23 # IDENT 24 u'äöü߀': [('IDENT', u'äöü߀', 1, 1)], 25 u' a ': [('S', u' ', 1, 1), 26 ('IDENT', u'a', 1, 2), 27 ('S', u' ', 1, 3)], 28 u'_a': [('IDENT', u'_a', 1, 1)], 29 u'-a': [('IDENT', u'-a', 1, 1)], 30 u'aA-_\200\377': [('IDENT', u'aA-_\200\377', 1, 1)], 31 u'a1': [('IDENT', u'a1', 1, 1)], 32 # escapes must end with S or max 6 digits: 33 u'\\1\nb': [('IDENT', u'\\1\nb', 1, 1)], 34 u'\\1\rb': [('IDENT', u'\\1\rb', 1, 1)], 35 u'\\1\fb': [('IDENT', u'\\1\fb', 1, 1)], 36 37 38 u'\\1 b': [('IDENT', u'\\1 b', 1, 1)], 39 u'\\12 b': [('IDENT', u'\\12 b', 1, 1)], 40 u'\\123 b': [('IDENT', u'\\123 b', 1, 1)], 41 u'\\1234 b': [('IDENT', u'\\1234 b', 1, 1)], 42 u'\\12345 b': [('IDENT', u'\\12345 b', 1, 1)], 43 u'\\123456 b': [('IDENT', u'\\123456 b', 1, 1)], 44 u'\\1234567 b': [('IDENT', u'\\1234567', 1, 1), 45 ('S', u' ', 1, 9), 46 ('IDENT', u'b', 1, 10)], 47 u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,': 48 [('IDENT', u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,', 1, 1)], 49 50 # STRING 51 u' "" ': [('S', u' ', 1, 1), 52 ('STRING', u'""', 1, 2), 53 ('S', u' ', 1, 4)], 54 u' "\'" ': [('S', u' ', 1, 1), 55 ('STRING', u'"\'"', 1, 2), 56 ('S', u' ', 1, 5)], 57 u" '' ": [('S', u' ', 1, 1), 58 ('STRING', u"''", 1, 2), 59 ('S', u' ', 1, 4)], 60 u" '' ": [('S', u' ', 1, 1), 61 ('STRING', u"''", 1, 2), 62 ('S', u' ', 1, 4)], 63 u"'\\\n'": [('STRING', u"'\\\n'", 1, 1)], 64 u"'\\\n\\\n\\\n'": [('STRING', u"'\\\n\\\n\\\n'", 1, 1)], 65 u"'\\\f'": [('STRING', u"'\\\f'", 1, 1)], 66 u"'\\\r'": [('STRING', u"'\\\r'", 1, 1)], 67 u"'\\\r\n'": [('STRING', u"'\\\r\n'", 1, 1)], 68 u"'1\\\n2'": [('STRING', u"'1\\\n2'", 1, 1)], 69 70 # HASH 71 u' #a ': [('S', u' ', 1, 1), 72 ('HASH', u'#a', 1, 2), 73 ('S', u' ', 1, 4)], 74 75 # NUMBER, for plus see CSS3 76 u' 0 ': [('S', u' ', 1, 1), 77 ('NUMBER', u'0', 1, 2), 78 ('S', u' ', 1, 3)], 79 u' 0.1 ': [('S', u' ', 1, 1), 80 ('NUMBER', u'0.1', 1, 2), 81 ('S', u' ', 1, 5)], 82 u' .0 ': [('S', u' ', 1, 1), 83 ('NUMBER', u'.0', 1, 2), 84 ('S', u' ', 1, 4)], 85 86 u' -0 ': [('S', u' ', 1, 1), 87 ('CHAR', u'-', 1, 2), 88 ('NUMBER', u'0', 1, 3), 89 ('S', u' ', 1, 4)], 90 91 # PERCENTAGE 92 u' 0% ': [('S', u' ', 1, 1), 93 ('PERCENTAGE', u'0%', 1, 2), 94 ('S', u' ', 1, 4)], 95 u' .5% ': [('S', u' ', 1, 1), 96 ('PERCENTAGE', u'.5%', 1, 2), 97 ('S', u' ', 1, 5)], 98 99 # URI 100 u' url() ': [('S', u' ', 1, 1), 101 ('URI', u'url()', 1, 2), 102 ('S', u' ', 1, 7)], 103 u' url(a) ': [('S', u' ', 1, 1), 104 ('URI', u'url(a)', 1, 2), 105 ('S', u' ', 1, 8)], 106 u' url("a") ': [('S', u' ', 1, 1), 107 ('URI', u'url("a")', 1, 2), 108 ('S', u' ', 1, 10)], 109 u' url( a ) ': [('S', u' ', 1, 1), 110 ('URI', u'url( a )', 1, 2), 111 ('S', u' ', 1, 10)], 112 u'ur\\l(': [('FUNCTION', u'ur\\l(', 1, 1)], 113 114 # UNICODE-RANGE 115 116 # CDO 117 u' <!-- ': [('S', u' ', 1, 1), 118 ('CDO', u'<!--', 1, 2), 119 ('S', u' ', 1, 6)], 120 u'"<!--""-->"': [('STRING', u'"<!--"', 1, 1), 121 ('STRING', u'"-->"', 1, 7)], 122 123 # CDC 124 u' --> ': [('S', u' ', 1, 1), 125 ('CDC', u'-->', 1, 2), 126 ('S', u' ', 1, 5)], 127 128 # S 129 u' ': [('S', u' ', 1, 1)], 130 u' ': [('S', u' ', 1, 1)], 131 u'\r': [('S', u'\r', 1, 1)], 132 u'\n': [('S', u'\n', 1, 1)], 133 u'\r\n': [('S', u'\r\n', 1, 1)], 134 u'\f': [('S', u'\f', 1, 1)], 135 u'\r': [('S', u'\r', 1, 1)], 136 u'\t': [('S', u'\t', 1, 1)], 137 u'\r\n\r\n\f\t ': [('S', u'\r\n\r\n\f\t ', 1, 1)], 138 139 # COMMENT, for incomplete see later 140 u'/*x*/ ': [('COMMENT', u'/*x*/', 1, 1), 141 ('S', u' ', 1, 6)], 142 143 # FUNCTION 144 u' x( ': [('S', u' ', 1, 1), 145 ('FUNCTION', u'x(', 1, 2), 146 ('S', u' ', 1, 4)], 147 # only url( is a valid URI so this must be a function 148 u'URL(': [('FUNCTION', u'URL(', 1, 1)], 149 u'uRl(': [('FUNCTION', u'uRl(', 1, 1)], 150 151 # INCLUDES 152 u' ~= ': [('S', u' ', 1, 1), 153 ('INCLUDES', u'~=', 1, 2), 154 ('S', u' ', 1, 4)], 155 u'~==': [('INCLUDES', u'~=', 1, 1), ('CHAR', u'=', 1, 3)], 156 157 # DASHMATCH 158 u' |= ': [('S', u' ', 1, 1), 159 ('DASHMATCH', u'|=', 1, 2), 160 ('S', u' ', 1, 4)], 161 u'|==': [('DASHMATCH', u'|=', 1, 1), ('CHAR', u'=', 1, 3)], 162 163 # CHAR 164 u' @ ': [('S', u' ', 1, 1), 165 ('CHAR', u'@', 1, 2), 166 ('S', u' ', 1, 3)], 167 168 # --- overwritten for CSS 2.1 --- 169 # LBRACE 170 u' { ': [('S', u' ', 1, 1), 171 ('CHAR', u'{', 1, 2), 172 ('S', u' ', 1, 3)], 173 # PLUS 174 u' + ': [('S', u' ', 1, 1), 175 ('CHAR', u'+', 1, 2), 176 ('S', u' ', 1, 3)], 177 # GREATER 178 u' > ': [('S', u' ', 1, 1), 179 ('CHAR', u'>', 1, 2), 180 ('S', u' ', 1, 3)], 181 # COMMA 182 u' , ': [('S', u' ', 1, 1), 183 ('CHAR', u',', 1, 2), 184 ('S', u' ', 1, 3)], 185 186 # class 187 u' . ': [('S', u' ', 1, 1), 188 ('CHAR', u'.', 1, 2), 189 ('S', u' ', 1, 3)], 190 191 } 192 193 tests3 = { 194 # ATKEYWORD 195 u' @x ': [('S', u' ', 1, 1), 196 ('ATKEYWORD', u'@x', 1, 2), 197 ('S', u' ', 1, 4)], 198 u'@X': [('ATKEYWORD', u'@X', 1, 1)], 199 u'@\\x': [('ATKEYWORD', u'@\\x', 1, 1)], 200 # - 201 u'@1x': [('CHAR', u'@', 1, 1), 202 ('DIMENSION', u'1x', 1, 2)], 203 204 # DIMENSION 205 u' 0px ': [('S', u' ', 1, 1), 206 ('DIMENSION', u'0px', 1, 2), 207 ('S', u' ', 1, 5)], 208 u' 1s ': [('S', u' ', 1, 1), 209 ('DIMENSION', u'1s', 1, 2), 210 ('S', u' ', 1, 4)], 211 u'0.2EM': [('DIMENSION', u'0.2EM', 1, 1)], 212 213 # NUMBER 214 u' - 0 ': [('S', u' ', 1, 1), 215 ('CHAR', u'-', 1, 2), 216 ('S', u' ', 1, 3), 217 ('NUMBER', u'0', 1, 4), 218 ('S', u' ', 1, 5)], 219 u' + 0 ': [('S', u' ', 1, 1), 220 ('CHAR', u'+', 1, 2), 221 ('S', u' ', 1, 3), 222 ('NUMBER', u'0', 1, 4), 223 ('S', u' ', 1, 5)], 224 225 # PREFIXMATCH 226 u' ^= ': [('S', u' ', 1, 1), 227 ('PREFIXMATCH', u'^=', 1, 2), 228 ('S', u' ', 1, 4)], 229 u'^==': [('PREFIXMATCH', u'^=', 1, 1), ('CHAR', u'=', 1, 3)], 230 231 # SUFFIXMATCH 232 u' $= ': [('S', u' ', 1, 1), 233 ('SUFFIXMATCH', u'$=', 1, 2), 234 ('S', u' ', 1, 4)], 235 u'$==': [('SUFFIXMATCH', u'$=', 1, 1), ('CHAR', u'=', 1, 3)], 236 237 # SUBSTRINGMATCH 238 u' *= ': [('S', u' ', 1, 1), 239 ('SUBSTRINGMATCH', u'*=', 1, 2), 240 ('S', u' ', 1, 4)], 241 u'*==': [('SUBSTRINGMATCH', u'*=', 1, 1), ('CHAR', u'=', 1, 3)], 242 243 # BOM 244 u' \xFEFF ': [('S', u' ', 1, 1), 245 ('BOM', u'\xFEFF', 1, 2), # len=3 246 ('S', u' ', 1, 5)], 247 248 } 249 250 tests2 = { 251 # escapes work not for a-f! 252 # IMPORT_SYM 253 u' @import ': [('S', u' ', 1, 1), 254 ('IMPORT_SYM', u'@import', 1, 2), 255 ('S', u' ', 1, 9)], 256 u'@IMPORT': [('IMPORT_SYM', u'@IMPORT', 1, 1)], 257 ur'@\i\m\p\o\r\t': [('IMPORT_SYM', ur'@\i\m\p\o\r\t', 1, 1)], 258 ur'@\I\M\P\O\R\T': [('IMPORT_SYM', ur'@\I\M\P\O\R\T', 1, 1)], 259 ur'@\49 \04d\0050\0004f\000052\54': [('IMPORT_SYM', 260 ur'@\49 \04d\0050\0004f\000052\54', 261 1, 1)], 262 ur'@\69 \06d\0070\0006f\000072\74': [('IMPORT_SYM', 263 ur'@\69 \06d\0070\0006f\000072\74', 264 1, 1)], 265 266 # PAGE_SYM 267 u' @page ': [('S', u' ', 1, 1), 268 ('PAGE_SYM', u'@page', 1, 2), 269 ('S', u' ', 1, 7)], 270 u'@PAGE': [('PAGE_SYM', u'@PAGE', 1, 1)], 271 ur'@\pa\ge': [('PAGE_SYM', ur'@\pa\ge', 1, 1)], 272 ur'@\PA\GE': [('PAGE_SYM', ur'@\PA\GE', 1, 1)], 273 ur'@\50\41\47\45': [('PAGE_SYM', ur'@\50\41\47\45', 1, 1)], 274 ur'@\70\61\67\65': [('PAGE_SYM', ur'@\70\61\67\65', 1, 1)], 275 276 # MEDIA_SYM 277 u' @media ': [('S', u' ', 1, 1), 278 ('MEDIA_SYM', u'@media', 1, 2), 279 ('S', u' ', 1, 8)], 280 u'@MEDIA': [('MEDIA_SYM', u'@MEDIA', 1, 1)], 281 ur'@\med\ia': [('MEDIA_SYM', ur'@\med\ia', 1, 1)], 282 ur'@\MED\IA': [('MEDIA_SYM', ur'@\MED\IA', 1, 1)], 283 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f': [('MEDIA_SYM', 284 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f', 285 1, 1)], 286 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f': [('MEDIA_SYM', 287 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f', 288 1, 1)], 289 290 # FONT_FACE_SYM 291 u' @font-face ': [('S', u' ', 1, 1), 292 ('FONT_FACE_SYM', u'@font-face', 1, 2), 293 ('S', u' ', 1, 12)], 294 u'@FONT-FACE': [('FONT_FACE_SYM', u'@FONT-FACE', 1, 1)], 295 ur'@f\o\n\t\-face': [('FONT_FACE_SYM', ur'@f\o\n\t\-face', 1, 1)], 296 ur'@F\O\N\T\-FACE': [('FONT_FACE_SYM', ur'@F\O\N\T\-FACE', 1, 1)], 297 # TODO: "-" as hex! 298 ur'@\46\4f\4e\54\-\46\41\43\45': [('FONT_FACE_SYM', 299 ur'@\46\4f\4e\54\-\46\41\43\45', 1, 1)], 300 ur'@\66\6f\6e\74\-\66\61\63\65': [('FONT_FACE_SYM', 301 ur'@\66\6f\6e\74\-\66\61\63\65', 1, 1)], 302 303 # CHARSET_SYM only if "@charset "! 304 u' @charset ': [('S', u' ', 1, 1), 305 ('CHARSET_SYM', u'@charset ', 1, 2), 306 ('S', u' ', 1, 11)], 307 u'@charset': [('ATKEYWORD', u'@charset', 1, 1)], # no ending S 308 u'@CHARSET ': [('ATKEYWORD', u'@CHARSET', 1, 1),# uppercase 309 ('S', u' ', 1, 9)], 310 u'@cha\\rset ': [('ATKEYWORD', u'@cha\\rset', 1, 1), # not literal 311 ('S', u' ', 1, 10)], 312 313 # NAMESPACE_SYM 314 u' @namespace ': [('S', u' ', 1, 1), 315 ('NAMESPACE_SYM', u'@namespace', 1, 2), 316 ('S', u' ', 1, 12)], 317 ur'@NAMESPACE': [('NAMESPACE_SYM', ur'@NAMESPACE', 1, 1)], 318 ur'@\na\me\s\pace': [('NAMESPACE_SYM', ur'@\na\me\s\pace', 1, 1)], 319 ur'@\NA\ME\S\PACE': [('NAMESPACE_SYM', ur'@\NA\ME\S\PACE', 1, 1)], 320 ur'@\4e\41\4d\45\53\50\41\43\45': [('NAMESPACE_SYM', 321 ur'@\4e\41\4d\45\53\50\41\43\45', 1, 1)], 322 ur'@\6e\61\6d\65\73\70\61\63\65': [('NAMESPACE_SYM', 323 ur'@\6e\61\6d\65\73\70\61\63\65', 1, 1)], 324 325 # ATKEYWORD 326 u' @unknown ': [('S', u' ', 1, 1), 327 ('ATKEYWORD', u'@unknown', 1, 2), 328 ('S', u' ', 1, 10)], 329 330 # STRING 331 # strings with linebreak in it 332 u' "\\na"\na': [('S', u' ', 1, 1), 333 ('STRING', u'"\\na"', 1, 2), 334 ('S', u'\n', 1, 7), 335 ('IDENT', u'a', 1, 8)], 336 u" '\\na'\na": [('S', u' ', 1, 1), 337 ('STRING', u"'\\na'", 1, 2), 338 ('S', u'\n', 1, 7), 339 ('IDENT', u'a', 1, 8)], 340 u' "\\r\\n\\t\\f\\n\\ra"a': [('S', u' ', 1, 1), 341 ('STRING', u'"\\r\\n\\t\\f\\n\\ra"', 1, 2), 342 ('IDENT', u'a', 1, 17)], 343 344 # IMPORTANT_SYM is not IDENT!!! 345 u' !important ': [('S', u' ', 1, 1), 346 ('CHAR', u'!', 1, 2), 347 ('IDENT', u'important', 1, 3), 348 ('S', u' ', 1, 12)], 349 u'! /*1*/ important ': [ 350 ('CHAR', u'!', 1, 1), 351 ('S', u' ', 1, 2), 352 ('COMMENT', u'/*1*/', 1, 3), 353 ('S', u' ', 1, 8), 354 ('IDENT', u'important', 1, 9), 355 ('S', u' ', 1, 18)], 356 u'! important': [('CHAR', u'!', 1, 1), 357 ('S', u' ', 1, 2), 358 ('IDENT', u'important', 1, 3)], 359 u'!\n\timportant': [('CHAR', u'!', 1, 1), 360 ('S', u'\n\t', 1, 2), 361 ('IDENT', u'important', 1, 4)], 362 u'!IMPORTANT': [('CHAR', u'!', 1, 1), 363 ('IDENT', u'IMPORTANT', 1, 2)], 364 ur'!\i\m\p\o\r\ta\n\t': [('CHAR', u'!', 1, 1), 365 ('IDENT', 366 ur'\i\m\p\o\r\ta\n\t', 1, 2)], 367 ur'!\I\M\P\O\R\Ta\N\T': [('CHAR', u'!', 1, 1), 368 ('IDENT', 369 ur'\I\M\P\O\R\Ta\N\T', 1, 2)], 370 ur'!\49\4d\50\4f\52\54\41\4e\54': [('CHAR', u'!', 1, 1), 371 ('IDENT', 372 ur'\49\4d\50\4f\52\54\41\4e\54', 373 1, 2)], 374 ur'!\69\6d\70\6f\72\74\61\6e\74': [('CHAR', u'!', 1, 1), 375 ('IDENT', 376 ur'\69\6d\70\6f\72\74\61\6e\74', 377 1, 2)], 378 } 379 380 # overwriting tests in testsall 381 tests2only = { 382 # LBRACE 383 u' { ': [('S', u' ', 1, 1), 384 ('LBRACE', u'{', 1, 2), 385 ('S', u' ', 1, 3)], 386 # PLUS 387 u' + ': [('S', u' ', 1, 1), 388 ('PLUS', u'+', 1, 2), 389 ('S', u' ', 1, 3)], 390 # GREATER 391 u' > ': [('S', u' ', 1, 1), 392 ('GREATER', u'>', 1, 2), 393 ('S', u' ', 1, 3)], 394 # COMMA 395 u' , ': [('S', u' ', 1, 1), 396 ('COMMA', u',', 1, 2), 397 ('S', u' ', 1, 3)], 398 # class 399 u' . ': [('S', u' ', 1, 1), 400 ('CLASS', u'.', 1, 2), 401 ('S', u' ', 1, 3)], 402 } 403 404 testsfullsheet = { 405 # TODO: escape ends with explicit space but \r\n as single space 406 #u'\\1\r\nb': [('IDENT', u'\\1\r', 1, 1), ('IDENT', u'b', 1, 4)], 407 408 # STRING 409 ur'"\" "': [('STRING', ur'"\" "', 1, 1)], 410 u"""'\\''""": [('STRING', u"""'\\''""", 1, 1)], 411 u'''"\\""''': [('STRING', u'''"\\""''', 1, 1)], 412 u' "\na': [('S', u' ', 1, 1), 413 ('INVALID', u'"', 1, 2), 414 ('S', u'\n', 1, 3), 415 ('IDENT', u'a', 1, 4)], 416 417 # strings with linebreak in it 418 u' "\\na\na': [('S', u' ', 1, 1), 419 ('INVALID', u'"\\na', 1, 2), 420 ('S', u'\n', 1, 6), 421 ('IDENT', u'a', 1, 7)], 422 u' "\\r\\n\\t\\f\\n\\ra\na': [('S', u' ', 1, 1), 423 ('INVALID', u'"\\r\\n\\t\\f\\n\\ra', 1, 2), 424 ('S', u'\n', 1, 16), 425 ('IDENT', u'a', 1, 17)], 426 } 427 428 # tests if fullsheet=False is set on tokenizer 429 testsfullsheetfalse = { 430 # COMMENT incomplete 431 u'/*': [('CHAR', u'/', 1, 1), 432 ('CHAR', u'*', 1, 2)], 433 434 # INVALID incomplete 435 u' " ': [('S', u' ', 1, 1), 436 ('INVALID', u'" ', 1, 2)], 437 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1), 438 ('INVALID', u"'abc\"with quote\" in it", 1, 2)], 439 440 # URI incomplete 441 u'url(a': [('FUNCTION', u'url(', 1, 1), 442 ('IDENT', u'a', 1, 5)], 443 u'url("a': [('FUNCTION', u'url(', 1, 1), 444 ('INVALID', u'"a', 1, 5)], 445 u"url('a": [('FUNCTION', u'url(', 1, 1), 446 ('INVALID', u"'a", 1, 5)], 447 } 448 449 # tests if fullsheet=True is set on tokenizer 450 testsfullsheettrue = { 451 # COMMENT incomplete 452 u'/*': [('COMMENT', u'/**/', 1, 1)], 453 454 # INVALID incomplete => STRING 455 u' " ': [('S', u' ', 1, 1), 456 ('STRING', u'" "', 1, 2)], 457 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1), 458 ('STRING', u"'abc\"with quote\" in it'", 1, 2)], 459 460 # URI incomplete FUNC => URI 461 u'url(a': [('URI', u'url(a)', 1, 1)], 462 u'url( a': [('URI', u'url( a)', 1, 1)], 463 u'url("a': [('URI', u'url("a")', 1, 1)], 464 u'url( "a ': [('URI', u'url( "a ")', 1, 1)], 465 u"url('a": [('URI', u"url('a')", 1, 1)], 466 u'url("a"': [('URI', u'url("a")', 1, 1)], 467 u"url('a'": [('URI', u"url('a')", 1, 1)], 468 469 } 470
471 - def setUp(self):
472 #log = cssutils.errorhandler.ErrorHandler() 473 self.tokenizer = Tokenizer()
474
475 - def test_linenumbers(self):
476 "Tokenizer line + col" 477 pass
478
479 - def test_tokenize(self):
480 "cssutils Tokenizer().tokenize()" 481 import cssutils.cssproductions 482 tokenizer = Tokenizer(cssutils.cssproductions.MACROS, 483 cssutils.cssproductions.PRODUCTIONS) 484 tests = {} 485 tests.update(self.testsall) 486 tests.update(self.tests2) 487 tests.update(self.tests3) 488 tests.update(self.testsfullsheet) 489 tests.update(self.testsfullsheetfalse) 490 for css in tests: 491 # check token format 492 tokens = tokenizer.tokenize(css) 493 for i, actual in enumerate(tokens): 494 expected = tests[css][i] 495 self.assertEqual(expected, actual) 496 497 # check if all same number of tokens 498 tokens = [t for t in tokenizer.tokenize(css)] 499 self.assertEqual(len(tokens), len(tests[css]))
500
501 - def test_tokenizefullsheet(self):
502 "cssutils Tokenizer().tokenize(fullsheet=True)" 503 import cssutils.cssproductions 504 tokenizer = Tokenizer(cssutils.cssproductions.MACROS, 505 cssutils.cssproductions.PRODUCTIONS) 506 tests = {} 507 tests.update(self.testsall) 508 tests.update(self.tests2) 509 tests.update(self.tests3) 510 tests.update(self.testsfullsheet) 511 tests.update(self.testsfullsheettrue) 512 for css in tests: 513 # check token format 514 tokens = tokenizer.tokenize(css, fullsheet=True) 515 for i, actual in enumerate(tokens): 516 try: 517 expected = tests[css][i] 518 except IndexError: 519 # EOF is added 520 self.assertEqual(actual[0], 'EOF') 521 else: 522 self.assertEqual(expected, actual) 523 524 # check if all same number of tokens 525 tokens = [t for t in tokenizer.tokenize(css, fullsheet=True)] 526 # EOF is added so -1 527 self.assertEqual(len(tokens) - 1, len(tests[css]))
528 529 530 # not really needed
531 - def test_tokenizeCSS3(self):
532 "CSS3 Tokenizer().tokenize()" 533 import cssutils.css3productions 534 tokenizer = Tokenizer(cssutils.css3productions.MACROS, 535 cssutils.css3productions.PRODUCTIONS) 536 tests = {} 537 tests.update(self.testsall) 538 tests.update(self.tests3) 539 for css in tests: 540 tokens = tokenizer.tokenize(css) 541 for i, actual in enumerate(tokens): 542 expected = tests[css][i] 543 self.assertEqual(expected, actual)
544 545 # not really needed
546 - def test_tokenizeCSS2_1(self):
547 "CSS2 Tokenizer().tokenize()" 548 import cssutils.css2productions 549 tokenizer = Tokenizer(cssutils.css2productions.MACROS, 550 cssutils.css2productions.PRODUCTIONS) 551 tests = {} 552 tests.update(self.testsall) 553 #tests.update(self.tests2) 554 tests.update(self.tests2only) 555 for css in tests: 556 tokens = tokenizer.tokenize(css) 557 for i, actual in enumerate(tokens): 558 expected = tests[css][i] 559 self.assertEqual(expected, actual)
560 561 # -------------- 562
563 - def __old(self):
564 565 testsOLD = { 566 u'x x1 -x .-x #_x -': [(1, 1, tt.IDENT, u'x'), 567 (1, 2, 'S', u' '), 568 (1, 3, tt.IDENT, u'x1'), 569 (1, 5, 'S', u' '), 570 (1, 6, tt.IDENT, u'-x'), 571 (1, 8, 'S', u' '), 572 (1, 9, tt.CLASS, u'.'), 573 (1, 10, tt.IDENT, u'-x'), 574 (1, 12, 'S', u' '), 575 (1, 13, tt.HASH, u'#_x'), 576 (1, 16, 'S', u' '), 577 (1, 17, 'DELIM', u'-')], 578 579 # num 580 u'1 1.1 -1 -1.1 .1 -.1 1.': [(1, 1, tt.NUMBER, u'1'), 581 (1, 2, 'S', u' '), (1, 3, tt.NUMBER, u'1.1'), 582 (1, 6, 'S', u' '), (1, 7, tt.NUMBER, u'-1'), 583 (1, 9, 'S', u' '), (1, 10, tt.NUMBER, u'-1.1'), 584 (1, 14, 'S', u' '), (1, 15, tt.NUMBER, u'0.1'), 585 (1, 17, 'S', u' '), (1, 18, tt.NUMBER, u'-0.1'), 586 (1, 21, 'S', u' '), 587 (1, 22, tt.NUMBER, u'1'), (1, 23, tt.CLASS, u'.') 588 ], 589 # CSS3 pseudo 590 u'::': [(1, 1, tt.PSEUDO_ELEMENT, u'::')], 591 592 # SPECIALS 593 u'*+>~{},': [(1, 1, tt.UNIVERSAL, u'*'), 594 (1, 2, tt.PLUS, u'+'), 595 (1, 3, tt.GREATER, u'>'), 596 (1, 4, tt.TILDE, u'~'), 597 (1, 5, tt.LBRACE, u'{'), 598 (1, 6, tt.RBRACE, u'}'), 599 (1, 7, tt.COMMA, u',')], 600 601 # DELIM 602 u'!%:&$|': [(1, 1, 'DELIM', u'!'), 603 (1, 2, 'DELIM', u'%'), 604 (1, 3, 'DELIM', u':'), 605 (1, 4, 'DELIM', u'&'), 606 (1, 5, 'DELIM', u'$'), 607 (1, 6, 'DELIM', u'|')], 608 609 610 # DIMENSION 611 u'5em': [(1, 1, tt.DIMENSION, u'5em')], 612 u' 5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'5em')], 613 u'5em ': [(1, 1, tt.DIMENSION, u'5em'), (1, 4, 'S', u' ')], 614 615 u'-5em': [(1, 1, tt.DIMENSION, u'-5em')], 616 u' -5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-5em')], 617 u'-5em ': [(1, 1, tt.DIMENSION, u'-5em'), (1, 5, 'S', u' ')], 618 619 u'.5em': [(1, 1, tt.DIMENSION, u'0.5em')], 620 u' .5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'0.5em')], 621 u'.5em ': [(1, 1, tt.DIMENSION, u'0.5em'), (1, 5, 'S', u' ')], 622 623 u'-.5em': [(1, 1, tt.DIMENSION, u'-0.5em')], 624 u' -.5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-0.5em')], 625 u'-.5em ': [(1, 1, tt.DIMENSION, u'-0.5em'), (1, 6, 'S', u' ')], 626 627 u'5em5_-': [(1, 1, tt.DIMENSION, u'5em5_-')], 628 629 u'a a5 a5a 5 5a 5a5': [(1, 1, tt.IDENT, u'a'), 630 (1, 2, 'S', u' '), 631 (1, 3, tt.IDENT, u'a5'), 632 (1, 5, 'S', u' '), 633 (1, 6, tt.IDENT, u'a5a'), 634 (1, 9, 'S', u' '), 635 (1, 10, tt.NUMBER, u'5'), 636 (1, 11, 'S', u' '), 637 (1, 12, tt.DIMENSION, u'5a'), 638 (1, 14, 'S', u' '), 639 (1, 15, tt.DIMENSION, u'5a5')], 640 641 # URI 642 u'url()': [(1, 1, tt.URI, u'url()')], 643 u'url();': [(1, 1, tt.URI, u'url()'), (1, 6, tt.SEMICOLON, ';')], 644 u'url("x")': [(1, 1, tt.URI, u'url("x")')], 645 u'url( "x")': [(1, 1, tt.URI, u'url("x")')], 646 u'url("x" )': [(1, 1, tt.URI, u'url("x")')], 647 u'url( "x" )': [(1, 1, tt.URI, u'url("x")')], 648 u' url("x")': [ 649 (1, 1, 'S', u' '), 650 (1, 2, tt.URI, u'url("x")')], 651 u'url("x") ': [ 652 (1, 1, tt.URI, u'url("x")'), 653 (1, 9, 'S', u' '), 654 ], 655 u'url(ab)': [(1, 1, tt.URI, u'url(ab)')], 656 u'url($#/ab)': [(1, 1, tt.URI, u'url($#/ab)')], 657 u'url(\1233/a/b)': [(1, 1, tt.URI, u'url(\1233/a/b)')], 658 # not URI 659 u'url("1""2")': [ 660 (1, 1, tt.FUNCTION, u'url('), 661 (1, 5, tt.STRING, u'"1"'), 662 (1, 8, tt.STRING, u'"2"'), 663 (1, 11, tt.RPARANTHESIS, u')'), 664 ], 665 u'url(a"2")': [ 666 (1, 1, tt.FUNCTION, u'url('), 667 (1, 5, tt.IDENT, u'a'), 668 (1, 6, tt.STRING, u'"2"'), 669 (1, 9, tt.RPARANTHESIS, u')'), 670 ], 671 u'url(a b)': [ 672 (1, 1, tt.FUNCTION, u'url('), 673 (1, 5, tt.IDENT, u'a'), 674 (1, 6, 'S', u' '), 675 (1, 7, tt.IDENT, u'b'), 676 (1, 8, tt.RPARANTHESIS, u')'), 677 ], 678 679 # FUNCTION 680 u' counter("x")': [ 681 (1,1, 'S', u' '), 682 (1, 2, tt.FUNCTION, u'counter('), 683 (1, 10, tt.STRING, u'"x"'), 684 (1, 13, tt.RPARANTHESIS, u')')], 685 # HASH 686 u'# #a #_a #-a #1': [ 687 (1, 1, 'DELIM', u'#'), 688 (1, 2, 'S', u' '), 689 (1, 3, tt.HASH, u'#a'), 690 (1, 5, 'S', u' '), 691 (1, 6, tt.HASH, u'#_a'), 692 (1, 9, 'S', u' '), 693 (1, 10, tt.HASH, u'#-a'), 694 (1, 13, 'S', u' '), 695 (1, 14, tt.HASH, u'#1') 696 ], 697 u'#1a1 ': [ 698 (1, 1, tt.HASH, u'#1a1'), 699 (1, 5, 'S', u' '), 700 ], 701 u'#1a1\n': [ 702 (1, 1, tt.HASH, u'#1a1'), 703 (1, 5, 'S', u'\n'), 704 ], 705 u'#1a1{': [ 706 (1, 1, tt.HASH, u'#1a1'), 707 (1, 5, tt.LBRACE, u'{'), 708 ], 709 u'#1a1 {': [ 710 (1, 1, tt.HASH, u'#1a1'), 711 (1, 5, 'S', u' '), 712 (1, 6, tt.LBRACE, u'{'), 713 ], 714 u'#1a1\n{': [ 715 (1, 1, tt.HASH, u'#1a1'), 716 (1, 5, 'S', u'\n'), 717 (2, 1, tt.LBRACE, u'{'), 718 ], 719 u'#1a1\n {': [ 720 (1, 1, tt.HASH, u'#1a1'), 721 (1, 5, 'S', u'\n '), 722 (2, 2, tt.LBRACE, u'{'), 723 ], 724 u'#1a1 \n{': [ 725 (1, 1, tt.HASH, u'#1a1'), 726 (1, 5, 'S', u' \n'), 727 (2, 1, tt.LBRACE, u'{'), 728 ], 729 # STRINGS with NL 730 u'"x\n': [(1,1, tt.INVALID, u'"x\n')], 731 u'"x\r': [(1,1, tt.INVALID, u'"x\r')], 732 u'"x\f': [(1,1, tt.INVALID, u'"x\f')], 733 u'"x\n ': [ 734 (1,1, tt.INVALID, u'"x\n'), 735 (2,1, 'S', u' ') 736 ] 737 738 }
739 740 # tests = { 741 # u'/*a': xml.dom.SyntaxErr, 742 # u'"a': xml.dom.SyntaxErr, 743 # u"'a": xml.dom.SyntaxErr, 744 # u"\\0 a": xml.dom.SyntaxErr, 745 # u"\\00": xml.dom.SyntaxErr, 746 # u"\\000": xml.dom.SyntaxErr, 747 # u"\\0000": xml.dom.SyntaxErr, 748 # u"\\00000": xml.dom.SyntaxErr, 749 # u"\\000000": xml.dom.SyntaxErr, 750 # u"\\0000001": xml.dom.SyntaxErr 751 # } 752 # self.tokenizer.log.raiseExceptions = True #!! 753 # for css, exception in tests.items(): 754 # self.assertRaises(exception, self.tokenizer.tokenize, css) 755 756 757 if __name__ == '__main__': 758 import unittest 759 unittest.main() 760