Package tlib :: Package base :: Module FileHelper
[hide private]
[frames] | no frames]

Source Code for Module tlib.base.FileHelper

  1  import paramiko 
  2  import urllib2 
  3  from Helper import Helper 
  4  from HelperException import FileNotExistException, BadFormatException, NotImplementedException 
  5   
  6  MAX_CHARS_IN_ONE_LINE = 20000 
  7   
8 -class SshFileHelper(Helper):
9 """ 10 Helper class for retrieving files from remote server 11 """ 12 _host = None 13 _username = None 14 _password = None 15 _pkey = None 16 _keyfile = None 17 18 _path = None 19 _size = None 20 _start = 0 21 22 _ssh = None 23 _client = None 24 _keepalive = None 25
26 - def __init__(self, host, username=None, password=None, pkey=None, key_filename=None, path=None, keep_alive=True):
27 """ 28 Constructor for class 29 30 @param host: remote hostname 31 @param username : login username 32 @param password : login password 33 @param pkey : private key used for login 34 @param key_filename : key file used for login 35 @param path : file path to be retrieved 36 @param keep_alive : establish only one connection 37 38 """ 39 Helper.__init__(self) 40 self._host = host 41 self._username = username 42 self._password = password 43 self._pkey = pkey 44 self._keyfile = key_filename 45 self._path = path 46 self._keepalive = keep_alive 47 self.ssh_connect()
48
49 - def __del__(self):
50 """ 51 Destructor for class 52 """ 53 self.ssh_disconnect()
54
55 - def ssh_connect(self):
56 """ 57 Establish ssh connection 58 """ 59 if not self._ssh or not self._keepalive: 60 self._ssh = paramiko.SSHClient() 61 self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 62 self._ssh.connect(hostname=self._host, username=self._username, password=self._password, pkey=self._pkey, key_filename=self._keyfile)
63
64 - def ssh_disconnect(self):
65 """ 66 Destroy ssh connection 67 """ 68 if self._client: 69 self._client.close() 70 self._client = None 71 if self._ssh: 72 self._ssh.close() 73 self._ssh = None
74
75 - def remote_tail(self, remote_host, path=None, length=0):
76 """ 77 Tail the file from a remote host (current connection used for hop purpose): 78 Please make sure the new ssh remote login does not require to input credentials 79 80 @param remote_host: target server where the file is located 81 @param path: path to file on the target server 82 @param length: length used for tailing 83 84 @return : a list of last n lines of file 85 """ 86 if path : 87 self._path = path 88 # make sure there's a connection 89 self.ssh_connect() 90 std_in, std_out, std_err = self._ssh.exec_command('ssh -t %s tail -%i %s' % (remote_host, length, self._path)) 91 lines = std_out.read().strip().split('\n') 92 return lines[-length:] if len(lines) > length else lines[1:]
93 94
95 - def tail(self, path=None, length=0):
96 """ 97 Tail a file on the server 98 99 @param path: path to file on the server 100 @param length: length used for tailing, tailing from last position (or beginning) if length is not provided 101 102 @return : a list of last n lines of file 103 """ 104 if path : 105 self._path = path 106 if not self._client: 107 # make sure there's a connection 108 self.ssh_connect() 109 self._client = self._ssh.open_sftp() 110 111 fstat = self._client.stat(self._path) 112 if length > 0: 113 #we assume that each line may contains MAX_CHARS_IN_ONE_LINE characters at max 114 self._start = max(0, fstat.st_size - (MAX_CHARS_IN_ONE_LINE * length)) 115 with self._client.open(self._path, 'r') as f: 116 # check if we have the file size 117 f.seek(self._start) 118 lines = f.read().strip().split('\n') 119 120 self._start = fstat.st_size 121 122 return lines[-length:] if len(lines) > length else lines[1:]
123 124
125 -class HttpFileHelper(Helper):
126 """ 127 Helper class for retrieving http files from url 128 """ 129 130 _url = None 131 _size = None 132
133 - def __init__(self, url=None):
134 """ 135 Constructor for class 136 """ 137 Helper.__init__(self) 138 self._url = url 139 self._size = self.getFileSize()
140
141 - def getFileSize(self):
142 """ 143 Get the size of file from the url 144 """ 145 with urllib2.urlopen(self._url) as fd: 146 info = fd.info() 147 return int(info.getheader('Content-Length'))
148 149
150 - def tail(self, url=None, length=0):
151 """ 152 Tail the file from the url 153 """ 154 if url: 155 self._url = url 156 self._size = self.getFileSize() 157 #we assume that each line may contains MAX_CHARS_IN_ONE_LINE characters at max 158 start = max(0, size - (MAX_CHARS_IN_ONE_LINE * length)) 159 #send the request asking for the last n bytes of the file content from the url 160 myHeaders = {'Range': 'bytes=%i-' % start} 161 req = urllib2.Request(url=self._url, headers=myHeaders) 162 partialFile = urllib2.urlopen(req) 163 164 lines = partialFile.read().strip().split('\n') 165 166 return lines[-length:] if len(lines) > length else lines[1:]
167