1 '''
2 Created on May 17, 2010
3
4 @author: dwmclary
5 '''
6
7 from . import hdfs_config as config
8 import subprocess
9
10
12 '''Generic runner for HDFS access'''
13 p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
14 r = p.communicate()
15 result = {"stdout":r[0], "stderr":r[1]}
16 return result
17
18
20 '''Runs HDFS cat on a filename or glob passed as a string
21 Returns the result as a string read from STOUT'''
22 args = [config.hadoop, "dfs", "-cat", filename]
23 return run(args)
24
25
27 '''Runs HDFS ls on a directory or glob passed as a string
28 Returns the result as a string read from STOUT'''
29 args = [config.hadoop, "dfs", "-ls", filename]
30 return run(args)
31
33 args = [config.hadoop, "dfs", "-tail", filename]
34 return run(args)
35
36 -def mv(filename1, filename2):
37 args = [config.hadoop, "dfs", "-mv", filename1, filename2]
38 return run(args)
39
41 '''Runs HDFS rmr on a filename or glob passed as a string
42 Returns the result as a string read from STOUT'''
43 args = [config.hadoop, "dfs", "-rmr", filename]
44 return run(args)
45
47 '''Runs HDFS rmr on a filename or glob passed as a string
48 Returns the result as a string read from STOUT'''
49 args = [config.hadoop, "dfs", "-rm", filename]
50 return run(args)
51
53 '''Runs HDFS mkdir on a directory name as a string
54 Returns the result as a string read from STOUT'''
55 args = [config.hadoop, "dfs", "-mkdir", dirname]
56 return run(args)
57
59 '''Runs HDFS rmr on a filename or glob passed as a string
60 Returns the result as a string read from STOUT'''
61 args = [config.hadoop, "dfs", "-getmerge", pattern, dst]
62 return run(args)
63
65 '''Runs HDFS mv: move src HDFS file to dst
66 Returns the result as a string read from STOUT'''
67 args = [config.hadoop, "dfs", "-mv", src, dst]
68 return run(args)
69
70
71
73 args = [config.hadoop, "dfs", "-put", filename, hdfs_filename]
74 return run(args)
75
77 args = [config.hadoop, "dfs", "-get", hdfs_filename, filename]
78 return run(args)
79