Source code for PyFoam.Applications.CompressCaseFiles

"""
Application-class that implements pyFoamCompressCaseFiles.py
"""
from optparse import OptionGroup
from os import path,listdir
import subprocess
import sys
from glob import glob

from .PyFoamApplication import PyFoamApplication

from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory

from PyFoam.ThirdParty.six import print_,PY3

from PyFoam.Basics.Utilities import humanReadableSize

if PY3:
    long=int

[docs]class CompressCaseFiles(PyFoamApplication): def __init__(self, args=None, **kwargs): description="""\ Gets a number of directories. If these are OpenFOAM-cases then it goes through them and checks for large uncompressed files and gnuzips them """ PyFoamApplication.__init__(self, args=args, description=description, usage="%prog [<directories>]", interspersed=True, changeVersion=False, nr=1, exactNr=False, **kwargs)
[docs] def addOptions(self): compress=OptionGroup(self.parser, "Compression", "Define what will be compressed") self.parser.add_option_group(compress) compress.add_option("--recursive", action="store_true", dest="recursive", default=False, help="Use the specified directories as a starting point and recursively search for cases") compress.add_option("--big-size", action="store", type="int", dest="bigSize", default=4095, help="Files with less bytes than this will not be compressed. Default: %default") compress.add_option("--compressed-extensions", action="append", dest="extensions", default=["gz","zip","tgz"], help="File extensions for which we assume that they are already compressed. Default: %default") compress.add_option("--logfiles", action="store_true", dest="logfile", default=False, help="Compress files in the case directory that end with .logfile (Assuming that these are logfiles generated by PyFoam)") feedback=OptionGroup(self.parser, "Feedback", "What should be printed") self.parser.add_option_group(feedback) feedback.add_option("--verbose", action="count", dest="verbose", default=1, help="Print names of the directories processed. Use multiple times for higher verbosity") feedback.add_option("--no-statistics", action="store_false", dest="statistics", default=True, help="Do not print a summary in the end") feedback.add_option("--silent", action="store_true", dest="silent", default=False, help="Switch off all output except warnings")
[docs] def compressFile(self,fName): if self.verbose>1: print_(" Compressing",fName) zippedName=fName+".gz" if path.exists(zippedName): self.warning("Zipped file",zippedName,"already existing for",fName) return oldSize=path.getsize(fName) if oldSize<self.bigSize: if self.verbose>2: print_(" Skipping because it is too small") self.nrSkipped+=1 return # use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program ret=subprocess.call(["gzip",fName]) if ret!=0 or not path.exists(zippedName) or path.exists(fName): self.warning("Problem compressing file",fName) self.nrProblems+=1 return newSize=path.getsize(zippedName) if newSize>oldSize: self.warning("Compression of",fName,"increased the filesize. Old:", humanReadableSize(oldSize),"New:",humanReadableSize(newSize)) if self.verbose>2: print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize)) self.nrFiles+=1 self.prevSize+=oldSize self.nowSize+=newSize
[docs] def compressDirectory(self,dirName): if self.verbose>1: print_(" Checking",dirName,"for compressible files") for f in listdir(dirName): if path.isdir(path.join(dirName,f)): self.compressDirectory(path.join(dirName,f)) else: name,ext=path.splitext(f) if ext.lower() not in self.extensions: self.compressFile(path.join(dirName,f)) else: self.nrCompressed+=1
[docs] def compressCase(self,dirName,warn=False): if not path.exists(dirName): self.error("Directory",dirName,"does not exist") s=SolutionDirectory(dirName, archive=None, paraviewLink=False, parallel=True, tolerant=True) if not s.isValid(): if warn: print_("Directory",dirName,"is not an OpenFOAM-case") return self.nrDir+=1 oldNr=self.nrFiles oldUnc=self.prevSize oldCon=self.nowSize if self.verbose>0: print_("Processing case",dirName) # compress meshes for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")): if path.isdir(d): self.compressDirectory(d) # compress times for t in s: self.compressDirectory(t.name) # compress logfiles if requested if self.opts.logfile: for f in glob(path.join(dirName,"*.logfile")): self.compressFile(path.join(dirName,f)) # processor direcories for p in s.procDirs: self.compressDirectory(path.join(dirName,p)) if self.nrFiles>oldNr and self.verbose>0: print_(" -> ",self.nrFiles-oldNr,"files compressed.", humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
[docs] def recursiveCompress(self,dirName): if self.verbose>1: print_("Recursively checking",dirName) if path.isdir(dirName): s=SolutionDirectory(dirName,archive=None,paraviewLink=False,parallel=True) if s.isValid(): try: self.compressCase(dirName) except OSError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' self.warning("Problem processing",dirName,":",e) return for f in listdir(dirName): name=path.join(dirName,f) try: if path.isdir(name): self.recursiveCompress(name) except OSError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' self.warning("Problem processing",name,":",e)
[docs] def run(self): dirs=self.parser.getArgs() self.bigSize=self.opts.bigSize self.verbose=self.opts.verbose self.extensions=["."+e.lower() for e in self.opts.extensions] if self.opts.silent: self.verbose=0 self.opts.statistics=False # for the statistics self.nrDir=0 self.nrCompressed=0 self.nrFiles=0 self.prevSize=0 self.nowSize=0 self.nrSkipped=0 self.nrProblems=0 try: for d in dirs: p=path.abspath(d) if self.opts.recursive: self.recursiveCompress(p) else: self.compressCase(p,warn=True) except KeyboardInterrupt: print_("Rudely interrupted by Control-C") if self.opts.statistics: if self.verbose>0: print_() print_(self.nrDir,"case directories processed") if self.nrFiles==0: print_("No files to compress found") else: print_(self.nrFiles,"files processed") print_("Reduced total size from",humanReadableSize(self.prevSize), "to",humanReadableSize(self.nowSize), ". This is",(100.*self.nowSize)/self.prevSize,"percent of the original") if self.nrSkipped>0: print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize) if self.nrCompressed>0: print_("Skipped",self.nrCompressed,"files because they are already compressed") if self.nrProblems>0: print_("Problems during the compression of",self.nrProblems,"files")
# Should work with Python3 and Python2