daily/2026-06-03/traces/140f57bfa29a4861e592b6723addfaac.json

{"processes":{"p0":{"serviceName":"pytest-observability","tags":[{"key":"cicd.pipeline.run.id","value":"26868669644:1"},{"key":"cicd.pipeline.task.name","value":"run"},{"key":"deployment.environment","value":"ci"},{"key":"process.command","value":"python3"},{"key":"process.command_args","value":"-m pytest -m is_tensor_parallel_test -n 1 -p no:warning -o junit_family=xunit1 --junitxml=test-results/junit.xml --reruns 5 --reruns-delay 2 --only-rerun (OSError|Timeout|ConnectionError|FileNotFoundError|PIL.UnidentifiedImageError|HTTPError|AssertionError: Tensor-likes are not close!|TypeError: expected str, bytes or os.PathLike object, not NoneType|TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType|Converting from Tiktoken failed|KeyError: <class |TypeError: not a string) --max-worker-restart=0 -vvv -rsfE --random-order-bucket=module --random-order-seed=2291 --make-reports=tests_tensor_parallel_ci tests/models/cwm/test_modeling_cwm.py tests/models/d_fine/test_modeling_d_fine.py tests/models/dab_detr/test_modeling_dab_detr.py tests/models/dac/test_modeling_dac.py tests/models/data2vec/test_modeling_data2vec_audio.py tests/models/data2vec/test_modeling_data2vec_text.py tests/models/data2vec/test_modeling_data2vec_vision.py tests/models/dbrx/test_modeling_dbrx.py tests/models/deberta/test_modeling_deberta.py tests/models/deberta_v2/test_modeling_deberta_v2.py tests/models/decision_transformer/test_modeling_decision_transformer.py tests/models/deepseek_ocr2/test_modeling_deepseek_ocr2.py tests/models/deepseek_v2/test_modeling_deepseek_v2.py tests/models/deepseek_v3/test_modeling_deepseek_v3.py tests/models/deepseek_v4/test_modeling_deepseek_v4.py tests/models/deepseek_vl/test_modeling_deepseek_vl.py tests/models/deepseek_vl_hybrid/test_modeling_deepseek_vl_hybrid.py tests/models/deformable_detr/test_modeling_deformable_detr.py tests/models/deimv2/test_modeling_deimv2.py tests/models/deit/test_modeling_deit.py tests/models/depth_anything/test_modeling_depth_anything.py tests/models/depth_pro/test_modeling_depth_pro.py tests/models/detr/test_modeling_detr.py tests/models/dia/test_modeling_dia.py tests/models/diffllama/test_modeling_diffllama.py tests/models/dinat/test_modeling_dinat.py tests/models/dinov2/test_modeling_dinov2.py tests/models/dinov2_with_registers/test_modeling_dinov2_"},{"key":"process.command_line","value":"python3 -m pytest -m is_tensor_parallel_test -n 1 -p no:warning -o junit_family=xunit1 --junitxml=test-results/junit.xml --reruns 5 --reruns-delay 2 --only-rerun (OSError|Timeout|ConnectionError|FileNotFoundError|PIL.UnidentifiedImageError|HTTPError|AssertionError: Tensor-likes are not close!|TypeError: expected str, bytes or os.PathLike object, not NoneType|TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType|Converting from Tiktoken failed|KeyError: <class |TypeError: not a string) --max-worker-restart=0 -vvv -rsfE --random-order-bucket=module --random-order-seed=2291 --make-reports=tests_tensor_parallel_ci tests/models/cwm/test_modeling_cwm.py tests/models/d_fine/test_modeling_d_fine.py tests/models/dab_detr/test_modeling_dab_detr.py tests/models/dac/test_modeling_dac.py tests/models/data2vec/test_modeling_data2vec_audio.py tests/models/data2vec/test_modeling_data2vec_text.py tests/models/data2vec/test_modeling_data2vec_vision.py tests/models/dbrx/test_modeling_dbrx.py tests/models/deberta/test_modeling_deberta.py tests/models/deberta_v2/test_modeling_deberta_v2.py tests/models/decision_transformer/test_modeling_decision_transformer.py tests/models/deepseek_ocr2/test_modeling_deepseek_ocr2.py tests/models/deepseek_v2/test_modeling_deepseek_v2.py tests/models/deepseek_v3/test_modeling_deepseek_v3.py tests/models/deepseek_v4/test_modeling_deepseek_v4.py tests/models/deepseek_vl/test_modeling_deepseek_vl.py tests/models/deepseek_vl_hybrid/test_modeling_deepseek_vl_hybrid.py tests/models/deformable_detr/test_modeling_deformable_detr.py tests/models/deimv2/test_modeling_deimv2.py tests/models/deit/test_modeling_deit.py tests/models/depth_anything/test_modeling_depth_anything.py tests/models/depth_pro/test_modeling_depth_pro.py tests/models/detr/test_modeling_detr.py tests/models/dia/test_modeling_dia.py tests/models/diffllama/test_modeling_diffllama.py tests/models/dinat/test_modeling_dinat.py tests/models/dinov2/test_modeling_dinov2.py tests/models/dinov2_with_registers/test_modeling"},{"key":"process.executable.name","value":"python3"},{"key":"process.executable.path","value":"/usr/local/bin/python3.10"},{"key":"process.owner","value":"root"},{"key":"process.pid","value":"634"},{"key":"process.runtime.description","value":"3.10.20 (main, May 19 2026, 23:52:13) [GCC 14.2.0]"},{"key":"process.runtime.name","value":"cpython"},{"key":"process.runtime.version","value":"3.10.20"},{"key":"service.version","value":"4c62cfff995cdc598a0b7d9b0232a99ed3a049b0"},{"key":"telemetry.sdk.language","value":"python"},{"key":"telemetry.sdk.name","value":"opentelemetry"},{"key":"telemetry.sdk.version","value":"1.42.1"},{"key":"transformers.test.job","value":"tests_tensor_parallel_ci"},{"key":"transformers.test.job.run","value":"26868669644:1:tests_tensor_parallel_ci"},{"key":"transformers.test.provider","value":"github_actions"},{"key":"transformers.test.run.id","value":"26868669644:1"},{"key":"vcs.change.id","value":"45604"},{"key":"vcs.change.url","value":"https://github.com/huggingface/transformers/pull/45604"},{"key":"vcs.provider.name","value":"github"},{"key":"vcs.ref.head.name","value":"45604/merge"},{"key":"vcs.ref.head.revision","value":"4c62cfff995cdc598a0b7d9b0232a99ed3a049b0"},{"key":"vcs.ref.type","value":"branch"},{"key":"vcs.repository.name","value":"huggingface/transformers"},{"key":"service.name","value":"pytest-observability"}]},"p1":{"serviceName":"pytest-observability","tags":[{"key":"cicd.pipeline.run.id","value":"26868669644:1"},{"key":"cicd.pipeline.task.name","value":"run"},{"key":"deployment.environment","value":"ci"},{"key":"process.command","value":"/usr/local/bin/python3"},{"key":"process.command_args","value":"-u -c import sys;exec(eval(sys.stdin.readline()))"},{"key":"process.command_line","value":"/usr/local/bin/python3 -u -c import sys;exec(eval(sys.stdin.readline()))"},{"key":"process.executable.name","value":"python3"},{"key":"process.executable.path","value":"/usr/local/bin/python3.10"},{"key":"process.owner","value":"root"},{"key":"process.pid","value":"647"},{"key":"process.runtime.description","value":"3.10.20 (main, May 19 2026, 23:52:13) [GCC 14.2.0]"},{"key":"process.runtime.name","value":"cpython"},{"key":"process.runtime.version","value":"3.10.20"},{"key":"service.version","value":"4c62cfff995cdc598a0b7d9b0232a99ed3a049b0"},{"key":"telemetry.sdk.language","value":"python"},{"key":"telemetry.sdk.name","value":"opentelemetry"},{"key":"telemetry.sdk.version","value":"1.42.1"},{"key":"transformers.test.job","value":"tests_tensor_parallel_ci"},{"key":"transformers.test.job.run","value":"26868669644:1:tests_tensor_parallel_ci"},{"key":"transformers.test.provider","value":"github_actions"},{"key":"transformers.test.run.id","value":"26868669644:1"},{"key":"vcs.change.id","value":"45604"},{"key":"vcs.change.url","value":"https://github.com/huggingface/transformers/pull/45604"},{"key":"vcs.provider.name","value":"github"},{"key":"vcs.ref.head.name","value":"45604/merge"},{"key":"vcs.ref.head.revision","value":"4c62cfff995cdc598a0b7d9b0232a99ed3a049b0"},{"key":"vcs.ref.type","value":"branch"},{"key":"vcs.repository.name","value":"huggingface/transformers"},{"key":"service.name","value":"pytest-observability"}]}},"spans":[{"duration":1968101695,"logs":[],"operationName":"test run","processID":"p0","startTime":1780469722989311,"tags":[{"key":"pytest.span_type","value":"run"},{"key":"otel.status_code","value":"OK"}]},{"duration":1937717620,"logs":[],"operationName":"test worker gw0","processID":"p1","startTime":1780469749315334,"tags":[{"key":"pytest.span_type","value":"run"},{"key":"otel.status_code","value":"OK"}]},{"duration":74171711,"logs":[],"operationName":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_ep_forward","processID":"p1","startTime":1780471224050948,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_ep_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"577"},{"key":"otel.status_code","value":"OK"}]},{"duration":74167993,"logs":[],"operationName":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_ep_forward::call","processID":"p1","startTime":1780471224053040,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_ep_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"577"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":242,"logs":[],"operationName":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_ep_forward::teardown","processID":"p1","startTime":1780471298221392,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_ep_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"577"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":135,"logs":[],"operationName":"_unittest_setUpClass_fixture_DeepseekV4ModelTest teardown","processID":"p1","startTime":1780471298221423,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_DeepseekV4ModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":558,"logs":[],"operationName":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_generation_quantized::setup","processID":"p1","startTime":1780471298224024,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":85,"logs":[],"operationName":"_unittest_setUpClass_fixture_DeepseekV3ModelTest setup","processID":"p1","startTime":1780471298224412,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_DeepseekV3ModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":162,"logs":[],"operationName":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_ep_backward::teardown","processID":"p1","startTime":1780471353215235,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_backward"},{"key":"pytest.nodeid","value":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_ep_backward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"593"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":71,"logs":[],"operationName":"_unittest_setUpClass_fixture_Ernie4_5_MoeModelTest teardown","processID":"p1","startTime":1780471353215258,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_Ernie4_5_MoeModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":34249,"logs":[],"operationName":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized","processID":"p1","startTime":1780471353229797,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"OK"}]},{"duration":561,"logs":[],"operationName":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized::setup","processID":"p1","startTime":1780471353260886,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":83,"logs":[],"operationName":"_unittest_setUpClass_fixture_DbrxModelTest setup","processID":"p1","startTime":1780471353261271,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_DbrxModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":1202,"logs":[],"operationName":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized::call","processID":"p1","startTime":1780471353261581,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":192,"logs":[],"operationName":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized::teardown","processID":"p1","startTime":1780471353263046,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/dbrx/test_modeling_dbrx.py::DbrxModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":93,"logs":[],"operationName":"_unittest_setUpClass_fixture_DbrxModelTest teardown","processID":"p1","startTime":1780471353263070,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_DbrxModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":64658075,"logs":[],"operationName":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_tp_forward","processID":"p1","startTime":1780471353264214,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_forward"},{"key":"pytest.nodeid","value":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_tp_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"505"},{"key":"otel.status_code","value":"OK"}]},{"duration":577,"logs":[],"operationName":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_tp_forward::setup","processID":"p1","startTime":1780471353320865,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_forward"},{"key":"pytest.nodeid","value":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_tp_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"505"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":81,"logs":[],"operationName":"_unittest_setUpClass_fixture_CwmModelTest setup","processID":"p1","startTime":1780471353321261,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_CwmModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":64598683,"logs":[],"operationName":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_tp_forward::call","processID":"p1","startTime":1780471353321579,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_forward"},{"key":"pytest.nodeid","value":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_tp_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"505"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":35482320,"logs":[],"operationName":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation","processID":"p1","startTime":1780471506468534,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation"},{"key":"pytest.nodeid","value":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"537"},{"key":"otel.status_code","value":"OK"}]},{"duration":35412123,"logs":[],"operationName":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation::call","processID":"p1","startTime":1780471506531929,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation"},{"key":"pytest.nodeid","value":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"537"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":4101,"logs":[],"operationName":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_ep_backward","processID":"p1","startTime":1780471572666972,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_backward"},{"key":"pytest.nodeid","value":"tests/models/cwm/test_modeling_cwm.py::CwmModelTest::test_ep_backward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"593"},{"key":"otel.status_code","value":"OK"}]},{"duration":533,"logs":[],"operationName":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_generation::setup","processID":"p1","startTime":1780471572710762,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_generation"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"537"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":100,"logs":[],"operationName":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_ep_backward::teardown","processID":"p1","startTime":1780471610842743,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_backward"},{"key":"pytest.nodeid","value":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_ep_backward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"593"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":12878243,"logs":[],"operationName":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation_quantized","processID":"p1","startTime":1780471610850623,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"OK"}]},{"duration":12826617,"logs":[],"operationName":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation_quantized::call","processID":"p1","startTime":1780471610901066,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/dots1/test_modeling_dots1.py::Dots1ModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":48805903,"logs":[],"operationName":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_forward","processID":"p1","startTime":1780469760154606,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"505"},{"key":"otel.status_code","value":"OK"}]},{"duration":1043,"logs":[],"operationName":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_forward::setup","processID":"p1","startTime":1780469760155505,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"505"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":174,"logs":[],"operationName":"event_loop_policy setup","processID":"p1","startTime":1780469760156128,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/pytest_asyncio/plugin.py"},{"key":"code.function","value":"event_loop_policy"},{"key":"code.lineno","value":"1076"},{"key":"pytest.fixture_scope","value":"session"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":49,"logs":[],"operationName":"_unittest_setUpClass_fixture_DeepseekV3ModelTest setup","processID":"p1","startTime":1780469760156397,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_DeepseekV3ModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":48798879,"logs":[],"operationName":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_forward::call","processID":"p1","startTime":1780469760156707,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3ModelTest::test_tp_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"505"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":50551,"logs":[],"operationName":"tests/models/deepseek_v2/test_modeling_deepseek_v2.py::DeepseekV2ModelTest::test_ep_forward","processID":"p1","startTime":1780470371917078,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_ep_forward"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v2/test_modeling_deepseek_v2.py::DeepseekV2ModelTest::test_ep_forward"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"577"},{"key":"otel.status_code","value":"OK"}]},{"duration":63732,"logs":[],"operationName":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_tp_generation","processID":"p1","startTime":1780470371967863,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_tp_generation"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"537"},{"key":"otel.status_code","value":"OK"}]},{"duration":175,"logs":[],"operationName":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_tp_generation::teardown","processID":"p1","startTime":1780470372021482,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation"},{"key":"pytest.nodeid","value":"tests/models/deepseek_v4/test_modeling_deepseek_v4.py::DeepseekV4ModelTest::test_tp_generation"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"537"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":271645736,"logs":[{"fields":[{"key":"event","value":"exception"},{"key":"exception.type","value":"Failed"},{"key":"exception.message","value":"Timeout (>120.0s) from pytest-timeout."},{"key":"exception.stacktrace","value":"self = <tests.models.ernie4_5_moe.test_modeling_ernie4_5_moe.Ernie4_5_MoeModelTest testMethod=test_tp_generation_quantized>\n\n    @is_tensor_parallel_test\n    def test_tp_generation_quantized(self):\n        self._skip_if_not_supported()\n    \n        if not is_torchao_available():\n            self.skipTest(\"Test requires torchao\")\n    \n        config = self.model_tester.get_config()\n        model_class = self._get_tp_model_class()\n        max_new_tokens = 25\n    \n        with tempfile.TemporaryDirectory() as tmp_dir:\n            set_seed(42)\n            model = model_class(config)\n            model.save_pretrained(tmp_dir, save_original_format=True)\n    \n>           _init_distributed(tp=self.tensor_parallel_size)(_test_tp_generation_quantized_impl)(\n                tmp_dir, model_class, max_new_tokens\n            )\n\ntests/test_tensor_parallel_mixin.py:574: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ntests/test_tensor_parallel_mixin.py:103: in wrapper\n    mp.spawn(_global_wrapper, args=spawn_args, nprocs=world_size)\n/usr/local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py:340: in spawn\n    return start_processes(fn, args, nprocs, join, daemon, start_method=\"spawn\")\n/usr/local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py:296: in start_processes\n    while not context.join():\n/usr/local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py:140: in join\n    ready = multiprocessing.connection.wait(\n/usr/local/lib/python3.10/multiprocessing/connection.py:931: in wait\n    ready = selector.select(timeout)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <selectors.PollSelector object at 0x7f4575c6b850>, timeout = None\n\n    def select(self, timeout=None):\n        # This is shared between poll() and epoll().\n        # epoll() has a different signature and handling of timeout parameter.\n        if timeout is None:\n            timeout = None\n        elif timeout <= 0:\n            timeout = 0\n        else:\n           "},{"key":"exception.escaped","value":"False"}]}],"operationName":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_tp_generation_quantized","processID":"p1","startTime":1780470372031867,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"OK"}]},{"duration":556,"logs":[],"operationName":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_tp_generation_quantized::setup","processID":"p1","startTime":1780470372060955,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":83,"logs":[],"operationName":"_unittest_setUpClass_fixture_Ernie4_5_MoeModelTest setup","processID":"p1","startTime":1780470372061339,"tags":[{"key":"code.filepath","value":"/usr/local/lib/python3.10/site-packages/_pytest/unittest.py"},{"key":"code.function","value":"_unittest_setUpClass_fixture_Ernie4_5_MoeModelTest"},{"key":"code.lineno","value":"139"},{"key":"pytest.fixture_scope","value":"class"},{"key":"pytest.span_type","value":"fixture"},{"key":"otel.status_code","value":"UNSET"}]},{"duration":119971286,"logs":[],"operationName":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_tp_generation_quantized::call","processID":"p1","startTime":1780470372061642,"tags":[{"key":"code.filepath","value":"tests/test_tensor_parallel_mixin.py"},{"key":"code.function","value":"test_tp_generation_quantized"},{"key":"pytest.nodeid","value":"tests/models/ernie4_5_moe/test_modeling_ernie4_5_moe.py::Ernie4_5_MoeModelTest::test_tp_generation_quantized"},{"key":"pytest.span_type","value":"test"},{"key":"code.lineno","value":"557"},{"key":"otel.status_code","value":"UNSET"}]}],"traceID":"140f57bfa29a4861e592b6723addfaac"}