Package lxml :: Package tests :: Module test_threading
[hide private]
[frames] | no frames]

Source Code for Module lxml.tests.test_threading

  1  # -*- coding: utf-8 -*- 
  2   
  3  """ 
  4  Tests for thread usage in lxml.etree. 
  5  """ 
  6   
  7  import unittest, threading, sys, os.path 
  8   
  9  this_dir = os.path.dirname(__file__) 
 10  if this_dir not in sys.path: 
 11      sys.path.insert(0, this_dir) # needed for Py3 
 12   
 13  from common_imports import etree, HelperTestCase, BytesIO, _bytes 
 14   
 15  try: 
 16      from Queue import Queue 
 17  except ImportError: 
 18      from queue import Queue # Py3 
 19   
20 -class ThreadingTestCase(HelperTestCase):
21 """Threading tests""" 22 etree = etree 23
24 - def _run_thread(self, func):
25 thread = threading.Thread(target=func) 26 thread.start() 27 thread.join()
28
29 - def test_subtree_copy_thread(self):
30 tostring = self.etree.tostring 31 XML = self.etree.XML 32 xml = _bytes("<root><threadtag/></root>") 33 main_root = XML(_bytes("<root/>")) 34 35 def run_thread(): 36 thread_root = XML(xml) 37 main_root.append(thread_root[0]) 38 del thread_root
39 40 self._run_thread(run_thread) 41 self.assertEquals(xml, tostring(main_root))
42
43 - def test_main_xslt_in_thread(self):
44 XML = self.etree.XML 45 style = XML(_bytes('''\ 46 <xsl:stylesheet version="1.0" 47 xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> 48 <xsl:template match="*"> 49 <foo><xsl:copy><xsl:value-of select="/a/b/text()" /></xsl:copy></foo> 50 </xsl:template> 51 </xsl:stylesheet>''')) 52 st = etree.XSLT(style) 53 54 result = [] 55 56 def run_thread(): 57 root = XML(_bytes('<a><b>B</b><c>C</c></a>')) 58 result.append( st(root) )
59 60 self._run_thread(run_thread) 61 self.assertEquals('''\ 62 <?xml version="1.0"?> 63 <foo><a>B</a></foo> 64 ''', 65 str(result[0])) 66
67 - def test_thread_xslt(self):
68 XML = self.etree.XML 69 tostring = self.etree.tostring 70 root = XML(_bytes('<a><b>B</b><c>C</c></a>')) 71 72 def run_thread(): 73 style = XML(_bytes('''\ 74 <xsl:stylesheet version="1.0" 75 xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> 76 <xsl:template match="*"> 77 <foo><xsl:copy><xsl:value-of select="/a/b/text()" /></xsl:copy></foo> 78 </xsl:template> 79 </xsl:stylesheet>''')) 80 st = etree.XSLT(style) 81 root.append( st(root).getroot() )
82 83 self._run_thread(run_thread) 84 self.assertEquals(_bytes('<a><b>B</b><c>C</c><foo><a>B</a></foo></a>'), 85 tostring(root)) 86
87 - def test_thread_create_xslt(self):
88 XML = self.etree.XML 89 tostring = self.etree.tostring 90 root = XML(_bytes('<a><b>B</b><c>C</c></a>')) 91 92 stylesheets = [] 93 94 def run_thread(): 95 style = XML(_bytes('''\ 96 <xsl:stylesheet 97 xmlns:xsl="http://www.w3.org/1999/XSL/Transform" 98 version="1.0"> 99 <xsl:output method="xml" /> 100 <xsl:template match="/"> 101 <div id="test"> 102 <xsl:apply-templates/> 103 </div> 104 </xsl:template> 105 </xsl:stylesheet>''')) 106 stylesheets.append( etree.XSLT(style) )
107 108 self._run_thread(run_thread) 109 110 st = stylesheets[0] 111 result = tostring( st(root) ) 112 113 self.assertEquals(_bytes('<div id="test">BC</div>'), 114 result) 115
116 - def test_thread_error_log(self):
117 XML = self.etree.XML 118 ParseError = self.etree.ParseError 119 expected_error = [self.etree.ErrorTypes.ERR_TAG_NAME_MISMATCH] 120 children = "<a>test</a>" * 100 121 122 def parse_error_test(thread_no): 123 tag = "tag%d" % thread_no 124 xml = "<%s>%s</%s>" % (tag, children, tag.upper()) 125 parser = self.etree.XMLParser() 126 for _ in range(10): 127 errors = None 128 try: 129 XML(xml, parser) 130 except self.etree.ParseError: 131 e = sys.exc_info()[1] 132 errors = e.error_log.filter_types(expected_error) 133 self.assert_(errors, "Expected error not found") 134 for error in errors: 135 self.assert_( 136 tag in error.message and tag.upper() in error.message, 137 "%s and %s not found in '%s'" % ( 138 tag, tag.upper(), error.message))
139 140 self.etree.clear_error_log() 141 threads = [] 142 for thread_no in range(1, 10): 143 t = threading.Thread(target=parse_error_test, 144 args=(thread_no,)) 145 threads.append(t) 146 t.start() 147 148 parse_error_test(0) 149 150 for t in threads: 151 t.join() 152
153 - def test_thread_mix(self):
154 XML = self.etree.XML 155 Element = self.etree.Element 156 SubElement = self.etree.SubElement 157 tostring = self.etree.tostring 158 xml = _bytes('<a><b>B</b><c xmlns="test">C</c></a>') 159 root = XML(xml) 160 fragment = XML(_bytes("<other><tags/></other>")) 161 162 result = self.etree.Element("{myns}root", att = "someval") 163 164 def run_XML(): 165 thread_root = XML(xml) 166 result.append(thread_root[0]) 167 result.append(thread_root[-1])
168 169 def run_parse(): 170 thread_root = self.etree.parse(BytesIO(xml)).getroot() 171 result.append(thread_root[0]) 172 result.append(thread_root[-1]) 173 174 def run_move_main(): 175 result.append(fragment[0]) 176 177 def run_build(): 178 result.append( 179 Element("{myns}foo", attrib={'{test}attr':'val'})) 180 SubElement(result, "{otherns}tasty") 181 182 def run_xslt(): 183 style = XML(_bytes('''\ 184 <xsl:stylesheet version="1.0" 185 xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> 186 <xsl:template match="*"> 187 <foo><xsl:copy><xsl:value-of select="/a/b/text()" /></xsl:copy></foo> 188 </xsl:template> 189 </xsl:stylesheet>''')) 190 st = etree.XSLT(style) 191 result.append( st(root).getroot()[0] ) 192 193 for test in (run_XML, run_parse, run_move_main, run_xslt): 194 tostring(result) 195 self._run_thread(test) 196 197 self.assertEquals( 198 _bytes('<ns0:root xmlns:ns0="myns" att="someval"><b>B</b><c xmlns="test">C</c><b>B</b><c xmlns="test">C</c><tags/><a>B</a></ns0:root>'), 199 tostring(result)) 200 201 def strip_first(): 202 root = Element("newroot") 203 root.append(result[0]) 204 205 while len(result): 206 self._run_thread(strip_first) 207 208 self.assertEquals( 209 _bytes('<ns0:root xmlns:ns0="myns" att="someval"/>'), 210 tostring(result)) 211
212 - def test_concurrent_proxies(self):
213 XML = self.etree.XML 214 root = XML(_bytes('<root><a>A</a><b xmlns="test">B</b><c/></root>')) 215 child_count = len(root) 216 def testrun(): 217 for i in range(10000): 218 el = root[i%child_count] 219 del el
220 threads = [ threading.Thread(target=testrun) 221 for _ in range(10) ] 222 for thread in threads: 223 thread.start() 224 for thread in threads: 225 thread.join() 226
227 - def test_concurrent_class_lookup(self):
228 XML = self.etree.XML 229 230 class TestElement(etree.ElementBase): 231 pass
232 233 class MyLookup(etree.CustomElementClassLookup): 234 repeat = range(100) 235 def lookup(self, t, d, ns, name): 236 count = 0 237 for i in self.repeat: 238 # allow other threads to run 239 count += 1 240 return TestElement 241 242 parser = self.etree.XMLParser() 243 parser.set_element_class_lookup(MyLookup()) 244 245 root = XML(_bytes('<root><a>A</a><b xmlns="test">B</b><c/></root>'), 246 parser) 247 248 child_count = len(root) 249 def testrun(): 250 for i in range(1000): 251 el = root[i%child_count] 252 del el 253 threads = [ threading.Thread(target=testrun) 254 for _ in range(10) ] 255 for thread in threads: 256 thread.start() 257 for thread in threads: 258 thread.join() 259 260
261 -class ThreadPipelineTestCase(HelperTestCase):
262 """Threading tests based on a thread worker pipeline. 263 """ 264 etree = etree 265 item_count = 20 266
267 - class Worker(threading.Thread):
268 - def __init__(self, in_queue, in_count, **kwargs):
269 threading.Thread.__init__(self) 270 self.in_queue = in_queue 271 self.in_count = in_count 272 self.out_queue = Queue(in_count) 273 self.__dict__.update(kwargs)
274 - def run(self):
275 get, put = self.in_queue.get, self.out_queue.put 276 handle = self.handle 277 for _ in range(self.in_count): 278 put(handle(get()))
279
280 - class ParseWorker(Worker):
281 XML = etree.XML
282 - def handle(self, xml):
283 return self.XML(xml)
284 - class RotateWorker(Worker):
285 - def handle(self, element):
286 first = element[0] 287 element[:] = element[1:] 288 element.append(first) 289 return element
290 - class ReverseWorker(Worker):
291 - def handle(self, element):
292 element[:] = element[::-1] 293 return element
294 - class ParseAndExtendWorker(Worker):
295 XML = etree.XML
296 - def handle(self, element):
297 element.extend(self.XML(self.xml)) 298 return element
299 - class SerialiseWorker(Worker):
300 - def handle(self, element):
301 return etree.tostring(element)
302 303 xml = _bytes('''\ 304 <xsl:stylesheet 305 xmlns:xsl="http://www.w3.org/1999/XSL/Transform" 306 version="1.0"> 307 <xsl:output method="xml" /> 308 <xsl:template match="/"> 309 <div id="test"> 310 <xsl:apply-templates/> 311 </div> 312 </xsl:template> 313 </xsl:stylesheet>''') 314
315 - def _build_pipeline(self, item_count, *classes, **kwargs):
316 in_queue = Queue(item_count) 317 start = last = classes[0](in_queue, item_count, **kwargs) 318 start.setDaemon(True) 319 for worker_class in classes[1:]: 320 last = worker_class(last.out_queue, item_count, **kwargs) 321 last.setDaemon(True) 322 last.start() 323 return (in_queue, start, last)
324
326 item_count = self.item_count 327 # build and start the pipeline 328 in_queue, start, last = self._build_pipeline( 329 item_count, 330 self.ParseWorker, 331 self.RotateWorker, 332 self.ReverseWorker, 333 self.ParseAndExtendWorker, 334 self.SerialiseWorker, 335 xml = self.xml) 336 337 # fill the queue 338 put = start.in_queue.put 339 for _ in range(item_count): 340 put(self.xml) 341 342 # start the first thread and thus everything 343 start.start() 344 # make sure the last thread has terminated 345 last.join(60) # time out after 60 seconds 346 self.assertEquals(item_count, last.out_queue.qsize()) 347 # read the results 348 get = last.out_queue.get 349 results = [ get() for _ in range(item_count) ] 350 351 comparison = results[0] 352 for i, result in enumerate(results[1:]): 353 self.assertEquals(comparison, result)
354
356 item_count = self.item_count 357 XML = self.etree.XML 358 # build and start the pipeline 359 in_queue, start, last = self._build_pipeline( 360 item_count, 361 self.RotateWorker, 362 self.ReverseWorker, 363 self.ParseAndExtendWorker, 364 self.SerialiseWorker, 365 xml = self.xml) 366 367 # fill the queue 368 put = start.in_queue.put 369 for _ in range(item_count): 370 put(XML(self.xml)) 371 372 # start the first thread and thus everything 373 start.start() 374 # make sure the last thread has terminated 375 last.join(60) # time out after 90 seconds 376 self.assertEquals(item_count, last.out_queue.qsize()) 377 # read the results 378 get = last.out_queue.get 379 results = [ get() for _ in range(item_count) ] 380 381 comparison = results[0] 382 for i, result in enumerate(results[1:]): 383 self.assertEquals(comparison, result)
384 385
386 -def test_suite():
387 suite = unittest.TestSuite() 388 suite.addTests([unittest.makeSuite(ThreadingTestCase)]) 389 suite.addTests([unittest.makeSuite(ThreadPipelineTestCase)]) 390 return suite
391 392 if __name__ == '__main__': 393 print('to test use test.py %s' % __file__) 394