1
2
3 from glob import glob
4 import sys, os, subprocess, shlex, random, time, re
5
7 wait_counter = 1
8 time.sleep(random.random())
9 all_checkpoints = ['0', '1', '2', '3', '4', '5', '6']
10 this_checkpoint = random.choice(all_checkpoints)
11 this_checkpoint_start = this_checkpoint+'_start'
12 this_checkpoint_end = this_checkpoint+'_end'
13 current_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*')
14 final_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*_end')
15 while len(final_checkpoints) < len(all_checkpoints):
16 for i in range(len(current_checkpoints)):
17 current_checkpoints[i] = re.sub('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/', '', current_checkpoints[i])
18 while this_checkpoint_end in current_checkpoints:
19 this_checkpoint = random.choice(all_checkpoints)
20 this_checkpoint_start = this_checkpoint+'_start'
21 this_checkpoint_end = this_checkpoint+'_end'
22 current_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*')
23 final_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*_end')
24 for i in range(len(current_checkpoints)):
25 current_checkpoints[i] = re.sub('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/', '', current_checkpoints[i])
26 if len(final_checkpoints) == len(all_checkpoints):
27 exit()
28
29 subprocess.call(['touch','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_start'])
30 subprocess.call(['chmod','777','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_start'])
31 os.system('chmod a+rwx fetch_books.py')
32 argf = open('./'+this_checkpoint).readlines()
33 for i in range(len(argf)):
34 argf[i] = argf[i].strip()
35 cmd = ['./fetch_books.py']+argf
36 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
37 output, error = p.communicate()
38 sts = p.wait()
39 output = output.strip()
40 if len(output) > 0:
41 line_count = 0
42 newline = '\n'
43 for line in output.split(newline):
44 print this_checkpoint+'==HDMC_CHECKPOINT==LINE=='+ str(line_count) + '=='+line.strip()
45 line_count += 1
46 if len(error.strip()) > 0:
47 print >> sys.stderr, error.strip()
48 os.system('rm /usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint)
49 exit(1)
50 else:
51 subprocess.call(['touch','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_end'])
52 subprocess.call(['chmod','777','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_end'])
53 os.system('rm /usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/'+this_checkpoint+'_start')
54 subprocess.call(['touch','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_end'])
55 this_checkpoint = random.choice(all_checkpoints)
56 current_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*')
57
58
59 if __name__ == "__main__":
60 main()
61