Package web2py :: Package gluon :: Module fileutils
[hide private]
[frames] | no frames]

Source Code for Module web2py.gluon.fileutils

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  This file is part of the web2py Web Framework 
  6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
  8  """ 
  9   
 10  import storage 
 11  import os 
 12  import re 
 13  import tarfile 
 14  import glob 
 15  from http import HTTP 
 16  from gzip import open as gzopen 
 17  from settings import global_settings 
 18   
 19   
 20  __all__ = [ 
 21      'up', 
 22      'abspath', 
 23      'listdir', 
 24      'recursive_unlink', 
 25      'cleanpath', 
 26      'tar', 
 27      'untar', 
 28      'tar_compiled', 
 29      'get_session', 
 30      'check_credentials', 
 31      'w2p_pack', 
 32      'w2p_unpack', 
 33      'w2p_pack_plugin', 
 34      'w2p_unpack_plugin', 
 35      'fix_newlines', 
 36      ] 
 37   
 38   
39 -def abspath(*relpath, **base):
40 "convert relative path to absolute path based (by default) on applications_parent" 41 path = os.path.join(*relpath) 42 gluon = base.get('gluon', False) 43 if os.path.isabs(path): 44 return path 45 if gluon: 46 return os.path.join(global_settings.gluon_parent, path) 47 return os.path.join(global_settings.applications_parent, path)
48
49 -def listdir( 50 path, 51 expression='^.+$', 52 drop=True, 53 add_dirs=False, 54 ):
55 """ 56 like os.listdir() but you can specify a regex pattern to filter files. 57 if add_dirs is True, the returned items will have the full path. 58 """ 59 60 if path[-1:] != '/': 61 path = path + '/' 62 if drop: 63 n = len(path) 64 else: 65 n = 0 66 regex = re.compile(expression) 67 items = [] 68 for (root, dirs, files) in os.walk(path, topdown=True): 69 for dir in dirs[:]: 70 if dir.startswith('.'): 71 dirs.remove(dir) 72 if add_dirs: 73 items.append(root[n:]) 74 for file in files: 75 if regex.match(file) and not file.startswith('.'): 76 items.append(os.path.join(root, file)[n:]) 77 return sorted(items)
78 79 87 88
89 -def cleanpath(path):
90 """ 91 turns any expression/path into a valid filename. replaces / with _ and 92 removes special characters. 93 """ 94 95 items = path.split('.') 96 if len(items) > 1: 97 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.' 98 + ''.join(items[-1:])) 99 else: 100 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:])) 101 return path
102 103
104 -def _extractall(filename, path='.', members=None):
105 if not hasattr(tarfile.TarFile, 'extractall'): 106 from tarfile import ExtractError 107 108 class TarFile(tarfile.TarFile): 109 110 def extractall(self, path='.', members=None): 111 """Extract all members from the archive to the current working 112 directory and set owner, modification time and permissions on 113 directories afterwards. `path' specifies a different directory 114 to extract to. `members' is optional and must be a subset of the 115 list returned by getmembers(). 116 """ 117 118 directories = [] 119 if members is None: 120 members = self 121 for tarinfo in members: 122 if tarinfo.isdir(): 123 124 # Extract directory with a safe mode, so that 125 # all files below can be extracted as well. 126 127 try: 128 os.makedirs(os.path.join(path, 129 tarinfo.name), 0777) 130 except EnvironmentError: 131 pass 132 directories.append(tarinfo) 133 else: 134 self.extract(tarinfo, path) 135 136 # Reverse sort directories. 137 138 directories.sort(lambda a, b: cmp(a.name, b.name)) 139 directories.reverse() 140 141 # Set correct owner, mtime and filemode on directories. 142 143 for tarinfo in directories: 144 path = os.path.join(path, tarinfo.name) 145 try: 146 self.chown(tarinfo, path) 147 self.utime(tarinfo, path) 148 self.chmod(tarinfo, path) 149 except ExtractError, e: 150 if self.errorlevel > 1: 151 raise 152 else: 153 self._dbg(1, 'tarfile: %s' % e)
154 155 156 _cls = TarFile 157 else: 158 _cls = tarfile.TarFile 159 160 tar = _cls(filename, 'r') 161 ret = tar.extractall(path, members) 162 tar.close() 163 return ret 164
165 -def tar(file, dir, expression='^.+$'):
166 """ 167 tars dir into file, only tars file that match expression 168 """ 169 170 tar = tarfile.TarFile(file, 'w') 171 for file in listdir(dir, expression, add_dirs=True): 172 tar.add(os.path.join(dir, file), file, False) 173 tar.close()
174
175 -def untar(file, dir):
176 """ 177 untar file into dir 178 """ 179 180 _extractall(file, dir)
181 182
183 -def w2p_pack(filename, path, compiled=False):
184 filename = abspath(filename) 185 path = abspath(path) 186 tarname = filename + '.tar' 187 if compiled: 188 tar_compiled(tarname, path, '^[\w\.\-]+$') 189 else: 190 tar(tarname, path, '^[\w\.\-]+$') 191 w2pfp = gzopen(filename, 'wb') 192 tarfp = open(tarname, 'rb') 193 w2pfp.write(tarfp.read()) 194 w2pfp.close() 195 tarfp.close() 196 os.unlink(tarname)
197
198 -def w2p_unpack(filename, path, delete_tar=True):
199 filename = abspath(filename) 200 path = abspath(path) 201 if filename[-4:] == '.w2p' or filename[-3:] == '.gz': 202 if filename[-4:] == '.w2p': 203 tarname = filename[:-4] + '.tar' 204 else: 205 tarname = filename[:-3] + '.tar' 206 fgzipped = gzopen(filename, 'rb') 207 tarfile = open(tarname, 'wb') 208 tarfile.write(fgzipped.read()) 209 tarfile.close() 210 fgzipped.close() 211 else: 212 tarname = filename 213 untar(tarname, path) 214 if delete_tar: 215 os.unlink(tarname)
216 217
218 -def w2p_pack_plugin(filename, path, plugin_name):
219 """Pack the given plugin into a w2p file. 220 Will match files at: 221 <path>/*/plugin_[name].* 222 <path>/*/plugin_[name]/* 223 """ 224 filename = abspath(filename) 225 path = abspath(path) 226 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): 227 raise Exception, "Not a web2py plugin name" 228 plugin_tarball = tarfile.open(filename, 'w:gz') 229 app_dir = path 230 while app_dir[-1]=='/': 231 app_dir = app_dir[:-1] 232 files1=glob.glob(os.path.join(app_dir,'*/plugin_%s.*' % plugin_name)) 233 files2=glob.glob(os.path.join(app_dir,'*/plugin_%s/*' % plugin_name)) 234 for file in files1+files2: 235 plugin_tarball.add(file, arcname=file[len(app_dir)+1:]) 236 plugin_tarball.close()
237 238
239 -def w2p_unpack_plugin(filename, path, delete_tar=True):
240 filename = abspath(filename) 241 path = abspath(path) 242 if not os.path.basename(filename).startswith('web2py.plugin.'): 243 raise Exception, "Not a web2py plugin" 244 w2p_unpack(filename,path,delete_tar)
245 246
247 -def tar_compiled(file, dir, expression='^.+$'):
248 """ 249 used to tar a compiled application. 250 the content of models, views, controllers is not stored in the tar file. 251 """ 252 253 tar = tarfile.TarFile(file, 'w') 254 for file in listdir(dir, expression, add_dirs=True): 255 filename = os.path.join(dir, file) 256 if os.path.islink(filename): 257 continue 258 if os.path.isfile(filename) and file[-4:] != '.pyc': 259 if file[:6] == 'models': 260 continue 261 if file[:5] == 'views': 262 continue 263 if file[:11] == 'controllers': 264 continue 265 if file[:7] == 'modules': 266 continue 267 tar.add(filename, file, False) 268 tar.close()
269
270 -def up(path):
271 return os.path.dirname(os.path.normpath(path))
272 273
274 -def get_session(request, other_application='admin'):
275 """ checks that user is authorized to access other_application""" 276 277 if request.application == other_application: 278 raise KeyError 279 try: 280 session_id = request.cookies['session_id_' + other_application].value 281 osession = storage.load_storage(os.path.join( 282 up(request.folder), other_application, 'sessions', session_id)) 283 except: 284 osession = storage.Storage() 285 return osession
286 287
288 -def check_credentials(request, other_application='admin'):
289 """ checks that user is authorized to access other_application""" 290 291 if request.env.web2py_runtime_gae: 292 from google.appengine.api import users 293 if users.is_current_user_admin(): 294 return True 295 else: 296 login_html = '<a href="%s">Sign in with your google account</a>.' \ 297 % users.create_login_url(request.env.path_info) 298 raise HTTP(200, '<html><body>%s</body></html>' % login_html) 299 else: 300 return get_session(request, other_application).authorized
301 302
303 -def fix_newlines(path):
304 regex = re.compile(r'''(\r 305 |\r| 306 )''') 307 for filename in listdir(path, '.*\.(py|html)$', drop=False): 308 fp = open(filename, 'rb') 309 rdata = fp.read() 310 fp.close() 311 wdata = regex.sub('\n', rdata) 312 if wdata != rdata: 313 fp = open(filename, 'wb') 314 fp.write(wdata) 315 fp.close()
316 317
318 -def copystream( 319 src, 320 dest, 321 size, 322 chunk_size=10 ** 5, 323 ):
324 """ 325 this is here because I think there is a bug in shutil.copyfileobj 326 """ 327 while size > 0: 328 if size < chunk_size: 329 data = src.read(size) 330 else: 331 data = src.read(chunk_size) 332 length = len(data) 333 if length > size: 334 (data, length) = (data[:size], size) 335 size -= length 336 if length == 0: 337 break 338 dest.write(data) 339 if length < chunk_size: 340 break 341 dest.seek(0) 342 return
343