%%time
treatment_doc_annotations = cat.get_entities_multi_texts(docs_filtered[‘body_analysed’].dropna(), n_process=24)
Exception raised when applying component meta_cat to a batch of docs.
Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the ‘spawn’ start method
Traceback (most recent call last):
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 376, in _set_meta_anns
all_predictions, all_confidences = predict(self.model, data, config)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/utils/meta_cat/ml_utils.py”, line 80, in predict
model.to(device)
File “/home/icat/.venv/play/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 852, in to
return self._apply(convert)
File “/home/icat/.venv/play/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 530, in _apply
module._apply(fn)
File “/home/icat/.venv/play/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 552, in _apply
param_applied = fn(param)
File “/home/icat/.venv/play/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 850, in convert
return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)
File “/home/icat/.venv/play/lib/python3.8/site-packages/torch/cuda/init.py”, line 162, in _lazy_init
raise RuntimeError(
RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the ‘spawn’ start method
Stack (most recent call last):
File “/opt/conda/lib/python3.8/runpy.py”, line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File “/opt/conda/lib/python3.8/runpy.py”, line 87, in _run_code
exec(code, run_globals)
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel_launcher.py”, line 16, in
app.launch_new_instance()
File “/home/icat/.venv/play/lib/python3.8/site-packages/traitlets/config/application.py”, line 846, in launch_instance
app.start()
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/kernelapp.py”, line 677, in start
self.io_loop.start()
File “/home/icat/.venv/play/lib/python3.8/site-packages/tornado/platform/asyncio.py”, line 199, in start
self.asyncio_loop.run_forever()
File “/opt/conda/lib/python3.8/asyncio/base_events.py”, line 570, in run_forever
self._run_once()
File “/opt/conda/lib/python3.8/asyncio/base_events.py”, line 1859, in _run_once
handle._run()
File “/opt/conda/lib/python3.8/asyncio/events.py”, line 81, in _run
self._context.run(self._callback, *self._args)
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/kernelbase.py”, line 461, in dispatch_queue
await self.process_one()
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/kernelbase.py”, line 450, in process_one
await dispatch(*args)
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/kernelbase.py”, line 357, in dispatch_shell
await result
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/kernelbase.py”, line 652, in execute_request
reply_content = await reply_content
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/ipkernel.py”, line 359, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File “/home/icat/.venv/play/lib/python3.8/site-packages/ipykernel/zmqshell.py”, line 532, in run_cell
return super().run_cell(*args, **kwargs)
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/interactiveshell.py”, line 2768, in run_cell
result = self._run_cell(
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/interactiveshell.py”, line 2814, in run_cell
return runner(coro)
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/async_helpers.py”, line 129, in pseudo_sync_runner
coro.send(None)
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/interactiveshell.py”, line 3012, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/interactiveshell.py”, line 3191, in run_ast_nodes
if await self.run_code(code, result, async=asy):
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/interactiveshell.py”, line 3251, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File “/tmp/ipykernel_281695/243332362.py”, line 1, in
get_ipython().run_cell_magic(‘time’, ‘’, “treatment_doc_annotations = cat.get_entities_multi_texts(docs_filtered[‘body_analysed’].dropna(), n_process=24)\n”)
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/interactiveshell.py”, line 2257, in run_cell_magic
result = fn(*args, **kwargs)
File “/home/icat/.venv/play/lib/python3.8/site-packages/IPython/core/magics/execution.py”, line 1316, in time
exec(code, glob, local_ns)
File “”, line 1, in
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/cat.py”, line 959, in get_entities_multi_texts
for doc in tqdm(docs, total=len(texts)):
File “/home/icat/.venv/play/lib/python3.8/site-packages/tqdm/notebook.py”, line 257, in iter
for obj in super(tqdm_notebook, self).iter():
File “/home/icat/.venv/play/lib/python3.8/site-packages/tqdm/std.py”, line 1180, in iter
for obj in iterable:
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/language.py”, line 1520, in pipe
for doc in docs:
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/language.py”, line 1556, in _multiprocessing_pipe
proc.start()
File “/opt/conda/lib/python3.8/multiprocessing/process.py”, line 121, in start
self._popen = self._Popen(self)
File “/opt/conda/lib/python3.8/multiprocessing/context.py”, line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File “/opt/conda/lib/python3.8/multiprocessing/context.py”, line 277, in _Popen
return Popen(process_obj)
File “/opt/conda/lib/python3.8/multiprocessing/popen_fork.py”, line 19, in init
self._launch(process_obj)
File “/opt/conda/lib/python3.8/multiprocessing/popen_fork.py”, line 75, in _launch
code = process_obj._bootstrap(parent_sentinel=child_r)
File “/opt/conda/lib/python3.8/multiprocessing/process.py”, line 315, in _bootstrap
self.run()
File “/opt/conda/lib/python3.8/multiprocessing/process.py”, line 108, in run
self._target(*self._args, **self._kwargs)
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/language.py”, line 2111, in _apply_pipes
byte_docs = [(doc.to_bytes(), None) for doc in docs]
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/language.py”, line 2111, in
byte_docs = [(doc.to_bytes(), None) for doc in docs]
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/util.py”, line 1526, in _pipe
for doc in docs:
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/util.py”, line 1516, in _pipe
yield from proc.pipe(docs, **kwargs)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 338, in pipe
yield from self._set_meta_anns(stream, batch_size_chars, config, id2category_value)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 348, in _set_meta_anns
for docs in self.batch_generator(stream, batch_size_chars):
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 308, in batch_generator
for doc in stream:
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/util.py”, line 1516, in _pipe
yield from proc.pipe(docs, **kwargs)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 338, in pipe
yield from self._set_meta_anns(stream, batch_size_chars, config, id2category_value)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 348, in _set_meta_anns
for docs in self.batch_generator(stream, batch_size_chars):
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 308, in batch_generator
for doc in stream:
File “/home/icat/.venv/play/lib/python3.8/site-packages/spacy/util.py”, line 1516, in _pipe
yield from proc.pipe(docs, **kwargs)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 338, in pipe
yield from self._set_meta_anns(stream, batch_size_chars, config, id2category_value)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/meta_cat.py”, line 401, in _set_meta_anns
self.get_error_handler()(self.name, self, docs, e)
File “/home/icat/.venv/play/lib/python3.8/site-packages/medcat/cat.py”, line 1424, in _pipe_error_handler
CAT.log.warning(e, exc_info=True, stack_info=True)