"""
Application-class that implements pyFoamCompressCaseFiles.py
"""
from optparse import OptionGroup
from os import path,listdir
import subprocess
import sys
from glob import glob
from .PyFoamApplication import PyFoamApplication
from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
from PyFoam.ThirdParty.six import print_,PY3
from PyFoam.Basics.Utilities import humanReadableSize
if PY3:
long=int
[docs]class CompressCaseFiles(PyFoamApplication):
def __init__(self,
args=None,
**kwargs):
description="""\
Gets a number of directories. If these are OpenFOAM-cases then it goes through them
and checks for large uncompressed files and gnuzips them
"""
PyFoamApplication.__init__(self,
args=args,
description=description,
usage="%prog [<directories>]",
interspersed=True,
changeVersion=False,
nr=1,
exactNr=False,
**kwargs)
[docs] def addOptions(self):
compress=OptionGroup(self.parser,
"Compression",
"Define what will be compressed")
self.parser.add_option_group(compress)
compress.add_option("--recursive",
action="store_true",
dest="recursive",
default=False,
help="Use the specified directories as a starting point and recursively search for cases")
compress.add_option("--big-size",
action="store",
type="int",
dest="bigSize",
default=4095,
help="Files with less bytes than this will not be compressed. Default: %default")
compress.add_option("--compressed-extensions",
action="append",
dest="extensions",
default=["gz","zip","tgz"],
help="File extensions for which we assume that they are already compressed. Default: %default")
compress.add_option("--logfiles",
action="store_true",
dest="logfile",
default=False,
help="Compress files in the case directory that end with .logfile (Assuming that these are logfiles generated by PyFoam)")
feedback=OptionGroup(self.parser,
"Feedback",
"What should be printed")
self.parser.add_option_group(feedback)
feedback.add_option("--verbose",
action="count",
dest="verbose",
default=1,
help="Print names of the directories processed. Use multiple times for higher verbosity")
feedback.add_option("--no-statistics",
action="store_false",
dest="statistics",
default=True,
help="Do not print a summary in the end")
feedback.add_option("--silent",
action="store_true",
dest="silent",
default=False,
help="Switch off all output except warnings")
[docs] def compressFile(self,fName):
if self.verbose>1:
print_(" Compressing",fName)
zippedName=fName+".gz"
if path.exists(zippedName):
self.warning("Zipped file",zippedName,"already existing for",fName)
return
oldSize=path.getsize(fName)
if oldSize<self.bigSize:
if self.verbose>2:
print_(" Skipping because it is too small")
self.nrSkipped+=1
return
# use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program
ret=subprocess.call(["gzip",fName])
if ret!=0 or not path.exists(zippedName) or path.exists(fName):
self.warning("Problem compressing file",fName)
self.nrProblems+=1
return
newSize=path.getsize(zippedName)
if newSize>oldSize:
self.warning("Compression of",fName,"increased the filesize. Old:",
humanReadableSize(oldSize),"New:",humanReadableSize(newSize))
if self.verbose>2:
print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize))
self.nrFiles+=1
self.prevSize+=oldSize
self.nowSize+=newSize
[docs] def compressDirectory(self,dirName):
if self.verbose>1:
print_(" Checking",dirName,"for compressible files")
for f in listdir(dirName):
if path.isdir(path.join(dirName,f)):
self.compressDirectory(path.join(dirName,f))
else:
name,ext=path.splitext(f)
if ext.lower() not in self.extensions:
self.compressFile(path.join(dirName,f))
else:
self.nrCompressed+=1
[docs] def compressCase(self,dirName,warn=False):
if not path.exists(dirName):
self.error("Directory",dirName,"does not exist")
s=SolutionDirectory(dirName,
archive=None,
paraviewLink=False,
parallel=True,
tolerant=True)
if not s.isValid():
if warn:
print_("Directory",dirName,"is not an OpenFOAM-case")
return
self.nrDir+=1
oldNr=self.nrFiles
oldUnc=self.prevSize
oldCon=self.nowSize
if self.verbose>0:
print_("Processing case",dirName)
# compress meshes
for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")):
if path.isdir(d):
self.compressDirectory(d)
# compress times
for t in s:
self.compressDirectory(t.name)
# compress logfiles if requested
if self.opts.logfile:
for f in glob(path.join(dirName,"*.logfile")):
self.compressFile(path.join(dirName,f))
# processor direcories
for p in s.procDirs:
self.compressDirectory(path.join(dirName,p))
if self.nrFiles>oldNr and self.verbose>0:
print_(" -> ",self.nrFiles-oldNr,"files compressed.",
humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
[docs] def recursiveCompress(self,dirName):
if self.verbose>1:
print_("Recursively checking",dirName)
if path.isdir(dirName):
s=SolutionDirectory(dirName,archive=None,paraviewLink=False,parallel=True)
if s.isValid():
try:
self.compressCase(dirName)
except OSError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
self.warning("Problem processing",dirName,":",e)
return
for f in listdir(dirName):
name=path.join(dirName,f)
try:
if path.isdir(name):
self.recursiveCompress(name)
except OSError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
self.warning("Problem processing",name,":",e)
[docs] def run(self):
dirs=self.parser.getArgs()
self.bigSize=self.opts.bigSize
self.verbose=self.opts.verbose
self.extensions=["."+e.lower() for e in self.opts.extensions]
if self.opts.silent:
self.verbose=0
self.opts.statistics=False
# for the statistics
self.nrDir=0
self.nrCompressed=0
self.nrFiles=0
self.prevSize=0
self.nowSize=0
self.nrSkipped=0
self.nrProblems=0
try:
for d in dirs:
p=path.abspath(d)
if self.opts.recursive:
self.recursiveCompress(p)
else:
self.compressCase(p,warn=True)
except KeyboardInterrupt:
print_("Rudely interrupted by Control-C")
if self.opts.statistics:
if self.verbose>0:
print_()
print_(self.nrDir,"case directories processed")
if self.nrFiles==0:
print_("No files to compress found")
else:
print_(self.nrFiles,"files processed")
print_("Reduced total size from",humanReadableSize(self.prevSize),
"to",humanReadableSize(self.nowSize),
". This is",(100.*self.nowSize)/self.prevSize,"percent of the original")
if self.nrSkipped>0:
print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize)
if self.nrCompressed>0:
print_("Skipped",self.nrCompressed,"files because they are already compressed")
if self.nrProblems>0:
print_("Problems during the compression of",self.nrProblems,"files")
# Should work with Python3 and Python2