Task 38486841

Name	wu_3c5123d1-GIANNI_GLLM-0-1-RND3995_2
Workunit	31482136
Created	23 Apr 2025, 13:14:59 UTC
Sent	23 Apr 2025, 13:20:43 UTC
Report deadline	28 Apr 2025, 13:20:43 UTC
Received	23 Apr 2025, 14:27:13 UTC
Server state	Over
Outcome	Computation error
Client state	Compute error
Exit status	195 (0x000000C3) EXIT_CHILD_FAILED
Computer ID	549890
Run time	10 min 59 sec
CPU time	10 min 59 sec
Validate state	Invalid
Credit	0.00
Device peak FLOPS	24,731.85 GFLOPS
Application version	LLM: LLMs for chemistry v1.00 (cuda124L) x86_64-pc-linux-gnu
Peak working set size	3.55 GB
Peak swap size	36.21 GB
Peak disk usage	8.12 GB

Stderr output

<core_client_version>7.19.0</core_client_version>
<![CDATA[
<message>
process exited with code 195 (0xc3, -61)</message>
<stderr_txt>
2025-04-23 10:21:20 (559546): wrapper (8.1.26018): starting
2025-04-23 10:22:39 (559546): wrapper: running bin/python (bin/conda-unpack)
2025-04-23 10:22:39 (559546): wrapper: created child process 559579
2025-04-23 10:22:42 (559546): bin/python exited; CPU time 1.596544
2025-04-23 10:22:42 (559546): wrapper: running bin/tar (xjvf input.tar.bz2)
2025-04-23 10:22:42 (559546): wrapper: created child process 559580
2025-04-23 10:22:43 (559546): bin/tar exited; CPU time 0.021282
2025-04-23 10:22:43 (559546): wrapper: running bin/bash (run.sh)
2025-04-23 10:22:43 (559546): wrapper: created child process 559582
+ echo 'Setup environment'
+ source bin/activate
++ _conda_pack_activate
++ local _CONDA_SHELL_FLAVOR
++ '[' -n x ']'
++ _CONDA_SHELL_FLAVOR=bash
++ local script_dir
++ case "$_CONDA_SHELL_FLAVOR" in
+++ dirname bin/activate
++ script_dir=bin
+++ cd bin
+++ pwd
++ local full_path_script_dir=/home/whitlerd/Desktop/BOINC/slots/90/bin
+++ dirname /home/whitlerd/Desktop/BOINC/slots/90/bin
++ local full_path_env=/home/whitlerd/Desktop/BOINC/slots/90
+++ basename /home/whitlerd/Desktop/BOINC/slots/90
++ local env_name=90
++ '[' -n '' ']'
++ export CONDA_PREFIX=/home/whitlerd/Desktop/BOINC/slots/90
++ CONDA_PREFIX=/home/whitlerd/Desktop/BOINC/slots/90
++ export _CONDA_PACK_OLD_PS1=
++ _CONDA_PACK_OLD_PS1=
++ PATH=/home/whitlerd/Desktop/BOINC/slots/90/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.
++ PS1='(90) '
++ case "$_CONDA_SHELL_FLAVOR" in
++ hash -r
++ local _script_dir=/home/whitlerd/Desktop/BOINC/slots/90/etc/conda/activate.d
++ '[' -d /home/whitlerd/Desktop/BOINC/slots/90/etc/conda/activate.d ']'
+ export PATH=/home/whitlerd/Desktop/BOINC/slots/90:/home/whitlerd/Desktop/BOINC/slots/90/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.
+ PATH=/home/whitlerd/Desktop/BOINC/slots/90:/home/whitlerd/Desktop/BOINC/slots/90/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.
+ echo 'Create a temporary directory'
+ export TMP=/home/whitlerd/Desktop/BOINC/slots/90/tmp
+ TMP=/home/whitlerd/Desktop/BOINC/slots/90/tmp
+ mkdir -p /home/whitlerd/Desktop/BOINC/slots/90/tmp
+ which python
+ pip install main_generation-0.1.0-py3-none-any.whl -v --no-deps
+ export CUDA_VISIBLE_DEVICES=0
+ CUDA_VISIBLE_DEVICES=0
+ export HF_HOME=../.cache
+ HF_HOME=../.cache
+ export VLLM_ASSETS_CACHE=../.cache
+ VLLM_ASSETS_CACHE=../.cache
+ export VLLM_CACHE_ROOT=../.cache
+ VLLM_CACHE_ROOT=../.cache
+ echo RUNNING
+ pythonbinary=/home/whitlerd/Desktop/BOINC/slots/90/lib/python3.12/site-packages/aiengine/main_generation.pyc
+ python /home/whitlerd/Desktop/BOINC/slots/90/lib/python3.12/site-packages/aiengine/main_generation.pyc --conf conf.yaml

Loading safetensors checkpoint shards:   0% Completed | 0/2 [00:00<?, ?it/s]

Loading safetensors checkpoint shards:  50% Completed | 1/2 [00:01<00:01,  1.50s/it]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:03<00:00,  1.53s/it]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:03<00:00,  1.53s/it]


Loading safetensors checkpoint shards:   0% Completed | 0/2 [00:00<?, ?it/s]

Loading safetensors checkpoint shards:  50% Completed | 1/2 [00:01<00:01,  1.67s/it]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:03<00:00,  1.69s/it]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:03<00:00,  1.68s/it]

run.sh: line 26: 559596 Killed                  python ${pythonbinary} --conf conf.yaml
2025-04-23 10:26:55 (559546): bin/bash exited; CPU time 59.609436
2025-04-23 10:26:55 (559546): app exit status: 0x89
2025-04-23 10:26:55 (559546): called boinc_finish(195)

</stderr_txt>
]]>