代码
import os
import sys
import time
cmd = 'nohup python -u train_post_2d_aut.py > output1.log &'
def gpu_info ( ) : gpu_status = os. popen( 'nvidia-smi | grep %' ) . read( ) . split( '|' ) gpu_memory = int ( gpu_status[ 10 ] . split( '/' ) [ 0 ] . split( 'M' ) [ 0 ] . strip( ) ) gpu_power = int ( gpu_status[ 9 ] . split( ' ' ) [ - 1 ] . split( '/' ) [ 0 ] . split( 'W' ) [ 0 ] . strip( ) ) power_all, memory_all = 250 , 12288 power_free = power_all - gpu_powermemory_free = memory_all - gpu_memoryreturn power_free, memory_free
def narrow_setup ( interval= 2 ) : power_free, memory_free = gpu_info( ) i = 0 while not ( memory_free > 7000 and power_free > 175 ) : power_free, memory_free = gpu_info( ) i = i % 10 symbol = 'monitoring: ' + '>' * i + ' ' * ( 10 - i - 1 ) + '|' gpu_power_str = 'gpu power_free:%d W |' % power_freegpu_memory_str = 'gpu memory_free:%d MiB |' % memory_freesys. stdout. write( '\r' + gpu_memory_str + ' ' + gpu_power_str + ' ' + symbol) sys. stdout. flush( ) time. sleep( interval) i += 1 print ( '\n' + cmd) os. system( cmd) if __name__ == '__main__' : narrow_setup( )