#
##
## This file is part of pyFormex 1.0.7 (Mon Jun 17 12:20:39 CEST 2019)
## pyFormex is a tool for generating, manipulating and transforming 3D
## geometrical models by sequences of mathematical operations.
## Home page: http://pyformex.org
## Project page: http://savannah.nongnu.org/projects/pyformex/
## Copyright 2004-2019 (C) Benedict Verhegghe (benedict.verhegghe@ugent.be)
## Distributed under the GNU General Public License version 3 or later.
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see http://www.gnu.org/licenses/.
##
"""A collection of miscellaneous utility functions.
The pyformex.utils module contains a wide variety of utilitary functions.
Because there are so many and they are so widely used, the utils module
is imported in the environment where scripts and apps are executed, so
that users can always call the utils functions without explicitely
importing the module.
"""
from __future__ import absolute_import, division, print_function
import os
import re
import tempfile
import time
import random
from collections import OrderedDict
import pyformex as pf
from pyformex import Path
from pyformex.process import Process
# These are here to re-export them as utils functions
from pyformex import (round, isFile, isString)
from pyformex.software import (hasModule, checkModule, requireModule,
hasExternal, checkExternal, requireExternal,
checkVersion)
from pyformex.config import formatDict
_exclude_members_ = ['runCommand']
shuffle = random.shuffle
# Some regular expressions
RE_digits = re.compile(r'(\d+)')
######### WARNINGS ##############
_warn_category = {'W': Warning, 'U': UserWarning, 'F': FutureWarning, 'D': DeprecationWarning}
def saveWarningFilter(message, module='', category=UserWarning):
pf.debug("Save {} '{}' '{}'".format(category.__name__, message, module), pf.DEBUG.WARNING)
cat = inverseDict(_warn_category).get(category, 'U')
oldfilters = pf.prefcfg['warnings/filters']
newfilters = oldfilters + [(str(message), '', cat)]
pf.prefcfg.update({'filters': newfilters}, name='warnings')
pf.debug("Warning filters: %s" % pf.prefcfg['warnings/filters'], pf.DEBUG.WARNING)
[docs]def filterWarning(message, module='', category='U', action='ignore'):
"""Add a warning message to the warnings filter.
category can be a Warning subclass or a key in the _warn_category dict
"""
import warnings
if not isinstance(category, Warning):
category = _warn_category.get(category, Warning)
pf.debug("Warning filter: {} {} '{}' from module '{}'".format(action, category.__name__, message, module), pf.DEBUG.WARNING)
warnings.filterwarnings(action, message, category, module)
def warn(message, level=UserWarning, stacklevel=3, data=None):
import warnings
from pyformex import messages
# pass the data into the message
messages._message_data = data
warnings.warn(message, level, stacklevel)
def deprec(message, stacklevel=4, data=None):
warn(message, level=FutureWarning, stacklevel=stacklevel, data=data)
[docs]def warning(message, level=UserWarning, stacklevel=3):
"""Decorator to add a warning to a function.
Adding this decorator to a function will warn the user with the
supplied message when the decorated function gets executed for
the first time in a session.
An option is provided to switch off this warning in future sessions.
Decorating a function is done as follows::
@utils.warning('This is the message shown to the user')
def function(args):
...
"""
import functools
def decorator(func):
def wrapper(*_args, **_kargs):
warn(message, level=level, stacklevel=stacklevel)
# For some reason these messages are not auto-appended to
# the filters for the currently running program
# Therefore we do it here explicitely
filterWarning(message, category=level)
return func(*_args, **_kargs)
#if level==UserWarning:
functools.update_wrapper(wrapper, func)
return wrapper
return decorator
[docs]def deprecated(message, stacklevel=4):
"""Decorator to deprecate a function
This is like :func:`warning`, but the level is set to FutureWarning.
"""
return warning(message, level=FutureWarning, stacklevel=stacklevel)
[docs]def deprecated_by(old, new, stacklevel=4):
"""Decorator to deprecate a function by another one.
Adding this decorator to a function will warn the user with a
message that the `old` function is deprecated in favor of `new`,
at the first execution of `old`.
See also: :func:`deprecated`.
"""
return deprecated("%s is deprecated: use %s instead" % (old, new), stacklevel=stacklevel)
[docs]def deprecated_future():
"""Decorator to warn that a function may be deprecated in future.
See also: :func:`deprecated`.
"""
return deprecated("This functionality is deprecated and will probably be removed in future, unless you explain to the developers why they should retain it.")
##########################################################################
## Running external commands ##
###############################
# Here are some pyFormex specific additions on top of the Process class
# The process of the last run command
last_command = None
[docs]def system(cmd, timeout=None, wait=True, verbose=False, raise_error=False, **kargs):
"""Execute an external command.
This and the more user oriented :func:`command` are the
prefered ways to execute external commands from inside pyFormex.
Parameters
----------
cmd: str
The command to be executed
timeout: float, optional
If specified and > 0.0, the command will time out and be terminated
or killed after the specified number of seconds.
wait: bool
If True (default), the caller waits for the process to terminate.
Setting this value to False will allow the caller to continue
immediately, but it will not be able to retrieve the standard output
and standard error of the process.
verbose: bool
If True, some extra informative messages are printed:
- the command that will be run,
- an occurring timeout condition,
- in case of a nonzero exit, the full stdout, exit status and stderr.
raise_error: bool.
If True, and verbose is True, and the command fails to execute or
returns with a nonzero return code other than one cause by a timeout,
an error is raised.
kargs: optional keyword parameters
Additional parameters that are passed to the Popen constructor.
See the Python documentation for full info.
Some often used parameters are below.
shell: bool
The default (False) is to run the command as a subprocess. This
has limitations however, as it will only accept parameters that
are processed by that command. Typically, you can not use
composite commands, I/O redirection, glob expansions.
With ``shell=True``, the command is run in a new shell.
The ``cmd`` should then be specified exactly as it would be
entered in a shell, and can contain anything the shell will accept.
stdout: open file object
If specified, the standard output of the command will
be written to that file.
stdin: open file object
If specified, the standard input of the command will
be read from that file.
Returns
-------
:class:`Process` object.
The Process used to run the command. This gives access to all
its info, like the exit code, stdout and stderr, and whether the command
timed out or not. See :class:`Process` for more info.
Notes
-----
The returned Process is also saved as the global variable
:attr:`last_command` to allow checking the outcome of the command
from other places but the caller.
"""
global last_command
if verbose:
print("Running command: %s" % cmd)
last_command = P = Process(cmd, timeout, wait, **kargs)
#P.run(timeout)
if verbose:
if P.timedout:
print("Command terminated due to timeout (%ss)" % timeout)
elif P.failed:
print("The subprocess failed to start, probably because the executable does not exist or is not in your current PATH.")
if raise_error:
raise RuntimeError("Error while executing command:\n %s" % cmd)
elif P.sta != 0:
print(P.out)
print("Command exited with an error (exitcode %s)" % P.sta)
print(P.err)
if raise_error:
raise RuntimeError("Error while executing command:\n %s" % cmd)
return P
[docs]def command(cmd, verbose=True, raise_error=True, **kargs):
"""Run an external command in a user friendly way.
This is equivalent with the :func:`system` function but has
verbose=True and raise_error=True options on by default.
"""
pf.debug("Command: %s" % cmd, pf.DEBUG.INFO)
return system(cmd, verbose=verbose, raise_error=raise_error, **kargs)
[docs]def lastCommandReport():
"""Produce a report about the last run external command.
Returns
-------
str
An extensive report about the last run command, including
output and error messages.
Notes
-----
This is mostly useful in interactive work, to find out
why a command failed.
"""
P = last_command
if P is None:
return 'No external command has been run yet'
s = "=" * 8 + " LAST COMMAND REPORT "
s += "=" * (50-len(s)) + "\n"
s += "%8s: %s\n" % ('cmd', P.cmd)
shell = P.kargs.get('shell', False)
if shell:
s += "%8s: %s\n" % ('shell', shell)
s += "%8s: %s\n" % ('status', P.sta)
for p in ['stdin', 'stdout', 'stderr']:
a = P.kargs.get(p, None)
name = a.name if isFile(a) else ''
txt = None
if p[3:] in ['out', 'err']:
txt = getattr(P, p[3:])
if txt is None and isFile(a):
try:
txt = open(a.name, 'r').read(2000)
except:
pass
if name or txt:
s += "%8s: %s\n" % (p, name)
if txt:
s += txt
if P.timeout:
s += "%8s: %s (timedout: %s)\n" % ('timeout', P.timeout, P.timedout)
s += "=" * 50 + "\n"
return s
[docs]def killProcesses(pids, signal=15):
"""Send the specified signal to the processes in list
Parameters
----------
pids: list of int
List of process ids to be killed.
signal: int
Signal to be send to the processes. The default (15) will
try to terminate the process in a friendly way.
See ``man kill`` for more values.
"""
for pid in pids:
try:
os.kill(pid, signal)
except:
pf.debug("Error in killing of process '%s'" % pid, pf.DEBUG.INFO)
[docs]def execSource(script, glob={}):
"""Execute Python code in another thread.
Parameters
----------
script: str
A string containing some executable Python/pyFormex code.
glob: dict, optional
A dict with globals specifying the environment in which the
source code is executed.
"""
pf.interpreter.locals = glob
pf.interpreter.runsource(script, '<input>', 'exec')
##########################################################################
## match multiple patterns
##########################
[docs]def matchMany(regexps, target):
"""Return multiple regular expression matches of the same target string."""
return [re.match(r, target) for r in regexps]
[docs]def matchCount(regexps, target):
"""Return the number of matches of target to regexps."""
return len([_f for _f in matchMany(regexps, target) if _f])
[docs]def matchAny(regexps, target):
"""Check whether target matches any of the regular expressions."""
return matchCount(regexps, target) > 0
[docs]def matchNone(regexps, target):
"""Check whether target matches none of the regular expressions."""
return matchCount(regexps, target) == 0
[docs]def matchAll(regexps, target):
"""Check whether targets matches all of the regular expressions."""
return matchCount(regexps, target) == len(regexps)
##########################################################################
## File types ##
################
# The pyFormex database of known file types.
# Only add file types here if they are relevan for pyFormex
file_description = OrderedDict([
('all', 'All files (*)'),
('ccx', 'CalCuliX files (*.dat *.inp)'),
('dcm', 'DICOM images (*.dcm)'),
('dxf', 'AutoCAD .dxf files (*.dxf)'),
('dxfall', 'AutoCAD .dxf or converted(*.dxf *.dxftext)'),
('dxftext', 'Converted AutoCAD files (*.dxftext)'),
('flavia', 'flavia results (*.flavia.msh *.flavia.res)'),
('gts', 'GTS files (*.gts)'),
('gz', 'Compressed files (*.gz *.bz2)'),
('html', 'Web pages (*.html)'),
('icon', 'Icons (*.xpm)'),
('img', 'Images (*.png *.jpg *.jpeg *.eps *.gif *.bmp)'),
('inp', 'Abaqus or CalCuliX input files (*.inp)'),
('neu', 'Gambit Neutral files (*.neu)'),
('obj', 'Wavefront OBJ files (*.obj)'),
('off', 'Geomview object files (*.off)'),
('pgf', 'pyFormex geometry files (*.pgf)'),
('ply', 'Stanford Polygon File Format files (*.ply)'),
('png', 'PNG images (*.png)'),
('postproc', 'Postproc scripts (*_post.py *.post)'),
('pyformex', 'pyFormex scripts (*.py *.pye)'),
('pyf', 'pyFormex projects (*.pyf)'),
('pzf', 'pyFormex zip files (*.pzf)'),
('smesh', 'Tetgen surface mesh files (*.smesh)'),
('stl', 'STL files (*.stl)'),
('stlb', 'Binary STL files (*.stl)'), # Use only for output
('surface', 'Surface models (*.off *.gts *.stl *.smesh *.vtp *.vtk)'),
('tetgen', 'Tetgen files (*.poly *.smesh *.ele *.face *.edge *.node *.neigh)'),
('vtk', 'All VTK types (*.vtk *.vtp)'),
('vtp', 'vtkPolyData file (*.vtp)'),
])
[docs]def fileDescription(ftype, compr=False):
"""Return a description of the specified file type(s).
Parameters
----------
ftype: str or list of str
The file type (or types) for which a description is requested.
The case of the string(s) is ignored: it is converted to lower case.
Returns
-------
str of list of str
The file description(s) corresponding with the specified file type(s).
The return value(s) depend(s) on the value of the input string(s) in the
the following way (see Examples below):
- if it is a key in the :attr:`file_description` dict, the
corresponding value is returned;
- if it is a string of only alphanumerical characters: it is interpreted
as a file extension and the corresponding return value is
``FTYPE files (*.ftype)``;
- any other string is returned as as: this allows the user to compose
his filters himself.
See also
--------
splitFileDescription
Examples
--------
>>> fileDescription('img')
'Images (*.png *.jpg *.jpeg *.eps *.gif *.bmp)'
>>> fileDescription(['stl','all'])
['STL files (*.stl)', 'All files (*)']
>>> fileDescription('inp')
'Abaqus or CalCuliX input files (*.inp)'
>>> fileDescription('doc')
'DOC files (*.doc)'
>>> fileDescription('*.inp')
'*.inp'
>>> fileDescription('pgf',compr=True)
'pyFormex geometry files (*.pgf *.pgf.gz *.pgf.bz2)'
This lists all known types in pyFormex:
>>> print(formatDict(file_description))
all = 'All files (*)'
ccx = 'CalCuliX files (*.dat *.inp)'
dcm = 'DICOM images (*.dcm)'
dxf = 'AutoCAD .dxf files (*.dxf)'
dxfall = 'AutoCAD .dxf or converted(*.dxf *.dxftext)'
dxftext = 'Converted AutoCAD files (*.dxftext)'
flavia = 'flavia results (*.flavia.msh *.flavia.res)'
gts = 'GTS files (*.gts)'
gz = 'Compressed files (*.gz *.bz2)'
html = 'Web pages (*.html)'
icon = 'Icons (*.xpm)'
img = 'Images (*.png *.jpg *.jpeg *.eps *.gif *.bmp)'
inp = 'Abaqus or CalCuliX input files (*.inp)'
neu = 'Gambit Neutral files (*.neu)'
obj = 'Wavefront OBJ files (*.obj)'
off = 'Geomview object files (*.off)'
pgf = 'pyFormex geometry files (*.pgf)'
ply = 'Stanford Polygon File Format files (*.ply)'
png = 'PNG images (*.png)'
postproc = 'Postproc scripts (*_post.py *.post)'
pyformex = 'pyFormex scripts (*.py *.pye)'
pyf = 'pyFormex projects (*.pyf)'
pzf = 'pyFormex zip files (*.pzf)'
smesh = 'Tetgen surface mesh files (*.smesh)'
stl = 'STL files (*.stl)'
stlb = 'Binary STL files (*.stl)'
surface = 'Surface models (*.off *.gts *.stl *.smesh *.vtp *.vtk)'
tetgen = 'Tetgen files (*.poly *.smesh *.ele *.face *.edge *.node *.neigh)'
vtk = 'All VTK types (*.vtk *.vtp)'
vtp = 'vtkPolyData file (*.vtp)'
<BLANKLINE>
"""
if isinstance(ftype, list):
return [fileDescription(f, compr) for f in ftype]
ftype = ftype.lower()
if ftype in file_description:
ret = file_description[ftype]
elif ftype.isalnum():
ret = "%s files (*.%s)" % (ftype.upper(), ftype)
else:
return ftype
if compr:
desc, ext = splitFileDescription(ret, compr)
ext = ['*'+e for e in ext]
ret = "%s (%s)" % (desc, ' '.join(ext))
return ret
[docs]def splitFileDescription(fdesc, compr=False):
"""Split a file descriptor.
A file descriptor is a string consisting of an initial part followed
by a second part enclosed in parentheses.
The second part is a space separated list of glob patterns. An example
file descriptor is 'file type text (\*.ext1 \*.ext2)'.
The values of :attr:`file_description` all have this format.
This function splits the file descriptor in two parts: the leading text
and a list of global patterns.
Parameters
----------
fdesc: str
A file descriptor string.
compr: bool,optional
If True, the compressed file types are automatically added.
Returns
-------
desc: str
The file type description text.
ext: list of str
A list of the matching extensions for this type. Each string
starts with a '.'.
See also
--------
fileDescription: return the file descriptor from file type
fileExtensionsFromFilter: return only the list of extensions
Examples
--------
>>> splitFileDescription(file_description['img'])
('Images ', ['.png', '.jpg', '.jpeg', '.eps', '.gif', '.bmp'])
>>> splitFileDescription(file_description['pgf'],compr=True)
('pyFormex geometry files ', ['.pgf', '.pgf.gz', '.pgf.bz2'])
"""
if pf.PY2 and isinstance(fdesc, unicode):
fdesc = fdesc.encode('utf-8')
desc, ext, dummy = re.compile('[()]').split(fdesc)
ext = [e.lstrip('*') for e in ext.split(' ')]
if compr:
ext = addCompressedTypes(ext)
return desc, ext
[docs]def fileExtensionsFromFilter(fdesc, compr=False):
"""Return the list of file extensions for a given file type.
Parameters
----------
fdesc: str
A file descriptor string.
compr: bool,optional
If True, the compressed file types are automatically added.
Returns
-------
list of str
A list of the matching extensions for this type. Each string
starts with a '.'.
Examples
--------
>>> fileExtensionsFromFilter(file_description['ccx'])
['.dat', '.inp']
>>> fileExtensionsFromFilter(file_description['ccx'],compr=True)
['.dat', '.dat.gz', '.dat.bz2', '.inp', '.inp.gz', '.inp.bz2']
"""
return splitFileDescription(fdesc, compr)[1]
[docs]def fileExtensions(ftype, compr=False):
"""Return the list of file extensions from a given type.
Parameters
----------
ftype: str
The file type (see :func:`fileDescription`.
compr: bool,optional
If True, the compressed file types are automatically added.
Returns
-------
list of str
A list of the matching extensions for this type. Each string
starts with a '.'.
Examples
--------
>>> fileExtensions('pgf')
['.pgf']
>>> fileExtensions('pgf',compr=True)
['.pgf', '.pgf.gz', '.pgf.bz2']
"""
return fileExtensionsFromFilter(file_description[ftype], compr)
[docs]def fileTypes(ftype, compr=False):
"""Return the list of file extension types for a given type.
Parameters
----------
ftype: str
The file type (see :func:`fileDescription`.
compr: bool,optional
If True, the compressed file types are automatically added.
Returns
-------
list of str
A list of the normalized matching extensions for this type.
Normalized extension do not have the leading dot and are
lower case only.
Examples
--------
>>> fileTypes('pgf')
['pgf']
>>> fileTypes('pgf',compr=True)
['pgf', 'pgf.gz', 'pgf.bz2']
"""
return [e.lower().lstrip('.') for e in fileExtensions(ftype, compr)]
[docs]def addCompressedTypes(ext):
"""Add the defined compress types to a list of extensions
Parameters
----------
ext: list of str
A list a filename extensions
Returns
-------
list of str
The list of filename extensions echo extended with the
corresponding compressed types.
Examples
--------
>>> addCompressedTypes(['.ccx','.inp'])
['.ccx', '.ccx.gz', '.ccx.bz2', '.inp', '.inp.gz', '.inp.bz2']
"""
from . import olist
ext = [[e] + [e+ec for ec in ['.gz', '.bz2']] for e in ext]
return olist.flatten(ext)
##########################################################################
## Filenames ##
###############
# TODO: these should go to path module
[docs]def projectName(fn):
"""Derive a project name from a file name.
The project name is the basename of the file without the extension.
It is equivalent with Path(fn).stem
Examples
--------
>>> projectName('aa/bb/cc.dd')
'cc'
>>> projectName('cc.dd')
'cc'
>>> projectName('cc')
'cc'
"""
return Path(fn).stem
[docs]def findIcon(name):
"""Return the file name for an icon with given name.
Parameters
----------
name: str
Name of the icon: this is the stem fof the filename.
Returns
-------
str
The full path name of an icon file with the specified name, found
in the pyFormex icon folder, or the question mark icon file, if
no match was found.
Examples
--------
>>> print(findIcon('view-xr-yu').relative_to(pf.cfg['pyformexdir']))
icons/view-xr-yu.xpm
>>> print(findIcon('right').relative_to(pf.cfg['pyformexdir']))
icons/64x64/right.png
>>> print(findIcon('xyz').relative_to(pf.cfg['pyformexdir']))
icons/question.xpm
"""
for icondir in pf.cfg['gui/icondirs']:
for icontype in pf.cfg['gui/icontypes']:
fname = icondir / (name+icontype)
if fname.exists():
return fname
return pf.cfg['icondir'] / 'question.xpm'
[docs]def listIconNames(dirs=None, types=None):
"""Return the list of available icons by their name.
Parameters
----------
dirs: list of paths, optional
If specified, only return icons names from these directories.
types: list of strings, optional
List of file suffixes, each starting with a dot. If specified,
Only names of icons having one of these suffixes are returned.
Returns
-------
list of str
A sorted list of the icon names available in the pyFormex icons folder.
Examples
--------
>>> listIconNames()[:4]
['clock', 'dist-angle', 'down', 'down']
>>> listIconNames([pf.cfg['icondir'] / '64x64'])[:4]
['down', 'ff', 'info', 'lamp']
>>> listIconNames(types=['.xpm'])[:4]
['clock', 'dist-angle', 'down', 'far']
"""
if dirs is None:
dirs = pf.cfg['gui/icondirs']
if types is None:
types = pf.cfg['gui/icontypes']
types = ['.+\\'+t for t in types]
icons = []
for icondir in dirs:
files = [Path(f).stem for f in icondir.files() if matchAny(types, f)]
icons.extend(files)
return sorted(icons)
##########################################################################
## File lists ##
################
[docs]def sourceFiles(relative=False, symlinks=True, extended=False):
"""Return the list of pyFormex .py source files.
Parameters
----------
relative: bool
If True, returned filenames are relative to the current directory.
symlinks: bool
If False, files that are symbolic links are retained in the
list. The default is to remove them.
extended: bool
If True, also return the .py files in all the paths in the configured
appdirs and scriptdirs.
Returns
-------
list of str
A list of filenames of .py files in the pyFormex source tree, and,
if ``extended`` is True, .py files in the configured app and script
dirs as well.
"""
path = pf.cfg['pyformexdir']
if relative:
path = path.relative_to(Path.cwd())
files = path.listTree(listdirs=False, sorted=True,
includedirs=pf.cfg['sourcedirs'],
includefiles=['.*\.py$', 'pyformexrc'], symlinks=symlinks)
if extended:
searchdirs = [Path(i[1]) for i in pf.cfg['appdirs'] + pf.cfg['scriptdirs']]
for path in set(searchdirs):
if path.exists():
files += path.listTree(listdirs=False, sorted=True,
includefiles=['.*\.py$'], symlinks=symlinks)
return files
[docs]def grepSource(pattern, options='', relative=True):
"""Finds pattern in the pyFormex source files.
Uses the `grep` program to find all occurrences of some specified
pattern text in the pyFormex source .py files (including the examples).
Extra options can be passed to the grep command. See `man grep` for
more info.
Returns the output of the grep command.
"""
opts = options.split(' ')
if '-a' in opts:
opts.remove('-a')
options = ' '.join(opts)
extended = True
else:
extended = False
files = sourceFiles(relative=relative, extended=extended, symlinks=False)
cmd = "grep %s '%s' %s" % (options, pattern, ' '.join(files))
P = system(cmd, verbose=True)
if not P.sta:
return P.out
[docs]def moduleList(package='all'):
"""Return a list of all pyFormex modules in a subpackage.
This is like :func:`sourceFiles`, but returns the files in a
Python module syntax.
"""
files = sourceFiles(relative=True)
dirname = files[0].parent
dirlen = len(dirname) + 1
modules = [fn[dirlen:].replace('.py', '').replace('/', '.') for fn in files]
if pf.PY3:
if 'compat_2k' in modules:
modules.remove('compat_2k')
if pf.PY2:
if 'compat_3k' in modules:
modules.remove('compat_3k')
if package == 'core':
# only core
modules = [m for m in modules if not '.' in m]
elif package == 'all':
# everything except examples
modules = [m for m in modules if not m.startswith('examples.') and not m.startswith('scripts.')]
elif package == 'all+ex':
# everything including examples
pass
else:
modules = [m for m in modules if m.startswith(package+'.')]
return modules
[docs]def diskSpace(path, units=None, ndigits=2):
"""Returns the amount of diskspace of a file system.
Parameters
----------
path: :term:`path_like`
A path name inside the file system to be probed.
units: str
If provided, results are reported in this units. See
:meth:`humanSize` for possible values. The default
is to return the number of bytes.
ndigits: int
If provided, and also ``units`` is provided, specifies the number
of decimal digits to report. See :meth:`humanSize` for details.
Returns
-------
total: int | float
The total disk space of the file system containing ``path``.
used: int | float
The used disk space on the file system containing ``path``.
available: int | float
The available disk space on the file system containing ``path``.
Notes
-----
The sum ``used + available`` does not necessarily equal ``total``,
because a file system may (and usually does) have reserved blocks.
"""
stat = os.statvfs(Path(path).resolve())
total = stat.f_blocks * stat.f_frsize
avail = stat.f_bavail * stat.f_frsize
used = (stat.f_blocks - stat.f_bfree) * stat.f_frsize
if units:
total = humanSize(total, units, ndigits)
avail = humanSize(avail, units, ndigits)
used = humanSize(used, units, ndigits)
return total, used, avail
[docs]def humanSize(size, units, ndigits=-1):
"""Convert a number to a human size.
Large numbers are often represented in a more human readable
form using k, M, G prefixes. This function returns the input
size as a number with the specified prefix.
Parameters
----------
size: int or float
A number to be converted to human readable form.
units: str
A string specifying the target units. The first character should
be one of k,K,M,G,T,P,E,Z,Y. 'k' and 'K' are equivalent. A second
character 'i' can be added to use binary (K=1024) prefixes instead of
decimal (k=1000).
ndigits: int, optional
If provided and >=0, the result will be rounded to this number of
decimal digits.
Returns
-------
float
The input value in the specified units and possibly rounded
to ``ndigits``.
Examples
--------
>>> humanSize(1234567890,'k')
1234567.89
>>> humanSize(1234567890,'M',0)
1235.0
>>> humanSize(1234567890,'G',3)
1.235
>>> humanSize(1234567890,'Gi',3)
1.15
"""
size = float(size)
order = '.KMGTPEZY'.find(units[0].upper())
if units[1:2] == 'i':
scale = 1024.
else:
scale = 1000.
size = size / scale**order
if ndigits >= 0:
size = round(size, ndigits)
return size
##########################################################################
## Temporary Files ##
#####################
if pf.PY2:
class TempDir(object):
"""A temporary directory that can be used as a context manager."""
def __init__(self, suffix='', prefix='tmp', dir=None):
self.path = Path(tempfile.mkdtemp(suffix, prefix, dir))
@property
def name(self):
return str(self.path)
def __repr__(self):
return "<{} {!r}>".format(self.__class__.__name__, self.name)
def __enter__(self):
return self.path
def __exit__(self, exc, value, tb):
self.cleanup()
def cleanup(self):
self.path.removeTree()
else:
[docs] class TempDir(tempfile.TemporaryDirectory):
"""A temporary directory that can be used as a context manager.
This is a wrapper around Python's tempfile.TemporaryDirectory.
The difference is that it has an extra .path attribute returning
the directory name as a Path, and the context manager also returns
a Path instead of a str.
"""
def __init__(self, suffix=None, prefix=None, dir=None):
super().__init__(suffix, prefix, dir)
@property
def path(self):
return Path(self.name)
def __enter__(self):
return self.path
[docs]def TempFile(*args, **kargs):
"""Return a temporary file that can be used as a context manager.
This is a wrapper around Python's tempfile.NamedTemporaryFile.
The difference is that the returned object has an extra .path
attribute holding the file name as a Path.
"""
tmpfile = tempfile.NamedTemporaryFile(*args, **kargs)
tmpfile.path = Path(tmpfile.name)
return tmpfile
##########################################################################
## ZIP Files
#####################
[docs]def zipList(filename):
"""List the files in a zip archive
Returns a list of file names
"""
from zipfile import ZipFile
zfil = ZipFile(filename, 'r')
return zfil.namelist()
###################### locale ###################
[docs]def setSaneLocale(localestring=''):
"""Set a sane local configuration for LC_NUMERIC.
`localestring` is the locale string to be set, e.g. 'en_US.UTF-8' or 'C'
for no locale.
Sets the ``LC_ALL`` locale to the specified string if that is not empty,
and (always) sets ``LC_NUMERIC`` and ``LC_COLLATE`` to 'C'.
Changing the LC_NUMERIC setting is a very bad idea! It makes floating
point values to be read or written with a comma instead of a the decimal
point. Of course this makes input and output files completely incompatible.
You will often not be able to process these files any further and
create a lot of troubles for yourself and other people if you use an
LC_NUMERIC setting different from the standard.
Because we do not want to help you shoot yourself in the foot, this
function always sets ``LC_NUMERIC`` back to a sane 'C' value and we
call this function when pyFormex is starting up.
"""
import locale
if localestring:
locale.setlocale(locale.LC_ALL, localestring)
locale.setlocale(locale.LC_NUMERIC, 'C')
locale.setlocale(locale.LC_COLLATE, 'C')
##########################################################################
## Text conversion tools ##
############################
[docs]def strNorm(s):
"""Normalize a string.
Text normalization removes all '&' characters and converts it to lower case.
>>> strNorm("&MenuItem")
'menuitem'
"""
return str(s).replace('&', '').lower()
_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.:]+')
[docs]def slugify(text, delim='-'):
"""Convert a string into a URL-ready readable ascii text.
Example:
>>> slugify("http://example.com/blog/[Some] _ Article's Title--")
'http-example-com-blog-some-article-s-title'
>>> slugify("&MenuItem")
'menuitem'
"""
import unicodedata
text = unicode(text)
result = []
for word in _punct_re.split(text.lower()):
word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore')
if word:
word = pf.to_str(word)
result.append(word)
return delim.join(result)
###################### ReST conversion ###################
#try:
# be quiet, because the import is done early
if checkModule('docutils', quiet=True):
from docutils.core import publish_string
def rst2html(text, writer='html'):
return publish_string(text, writer_name=writer)
#except ImportError:
else:
def rst2html(text, writer='html'):
return """.. note:
This is a reStructuredText message, but it is currently displayed
as plain text, because it could not be converted to html.
If you install python-docutils, you will see this text (and other
pyFormex messages) in a much nicer layout!
""" + text
[docs]def forceReST(text, underline=False):
"""Convert a text string to have it recognized as reStructuredText.
Returns the text with two lines prepended: a line with '..'
and a blank line. The text display functions will then recognize the
string as being reStructuredText. Since the '..' starts a comment in
reStructuredText, it will not be displayed.
Furthermore, if `underline` is set True, the first line of the text
will be underlined to make it appear as a header.
"""
if underline:
text = underlineHeader(text)
return "..\n\n" + text
[docs]def framedText(text, padding=[0, 2, 0, 2], border=[1, 2, 1, 2], margin=[0, 0, 0, 0],
borderchar='####', cornerchar=None, width=None, adjust='l'):
"""Create a text with a frame around it.
- `adjust`: 'l', 'c' or 'r': makes the text lines be adjusted to the
left, center or right.
>>> print(framedText("Hello World,\\nThis is me calling",adjust='c'))
##########################
## Hello World, ##
## This is me calling ##
##########################
"""
lines = text.splitlines()
maxlen = max([len(l) for l in lines])
if adjust == 'c':
lines = [l.center(maxlen) for l in lines]
elif adjust == 'r':
lines = [l.rjust(maxlen) for l in lines]
else:
lines = [l.ljust(maxlen) for l in lines]
prefix = ' '*margin[3] + borderchar[3]*border[3] + ' '*padding[3]
suffix = ' '*padding[1] + borderchar[1]*border[1] + ' '*margin[1]
width = len(prefix)+maxlen+len(suffix)
s = []
for i in range(margin[0]):
s.append('')
for i in range(border[0]):
s.append(borderchar[0]*width)
for i in range(padding[0]):
s.append(prefix+' '*maxlen+suffix)
for l in lines:
s.append(prefix+("%"+str(maxlen)+"s") % l+suffix)
for i in range(padding[2]):
s.append(prefix+' '*maxlen+suffix)
for i in range(border[2]):
s.append(borderchar[2]*width)
for i in range(margin[2]):
s.append('')
return '\n'.join(s)
[docs]def prefixText(text, prefix):
"""Add a prefix to all lines of a text.
- `text`: multiline string
- `prefix`: string: prefix to insert at the start of all lines of text.
>>> print(prefixText("line1\\nline2","** "))
** line1
** line2
"""
return '\n'.join([prefix+line for line in text.split('\n')])
[docs]def versaText(obj):
"""Versatile text creator
This functions converts any input into a (multiline) string.
Currently, different inputs are treated as follows:
- string: return as is,
- function: use the output of the function as input,
- anything else: use str() or repr() on the object.
>>> versaText("Me")
'Me'
>>> versaText(1)
'1'
>>> versaText(len("Me"))
'2'
>>> versaText({1:"Me"})
"{1: 'Me'}"
"""
if callable(obj):
obj = obj()
if not isinstance(obj, str):
try:
obj = str(obj)
except:
obj = repr(obj)
return obj
###################### file conversion ###################
[docs]def dos2unix(infile):
"""Convert a text file to unix line endings."""
return system("sed -i 's|$|\\r|' %s" % infile)
[docs]def unix2dos(infile, outfile=None):
"""Convert a text file to dos line endings."""
return system("sed -i 's|\\r||' %s" % infile)
[docs]def gzip(filename, gzipped=None, remove=True, level=5, compr='gz'):
"""Compress a file in gzip/bzip2 format.
Parameters:
- `filename`: input file name
- `gzipped`: output file name. If not specified, it will be set to
the input file name + '.' + `compr`. An existing output file will be
overwritten.
- `remove`: if True (default), the input file is removed after
succesful compression
- `level`: an integer from 1..9: gzip/bzip2 compression level.
Higher values result in smaller files, but require longer compression
times. The default of 5 gives already a fairly good compression ratio.
- `compr`: 'gz' or 'bz2': the compression algorithm to be used.
The default is 'gz' for gzip compression. Setting to 'bz2' will use
bzip2 compression.
Returns the name of the compressed file.
"""
filename = str(filename)
if gzipped is None:
gzipped = filename+'.'+compr
if compr == 'gz':
import gzip
gz = gzip.GzipFile(gzipped, 'wb', compresslevel=level)
elif compr == 'bz2':
import bz2
gz = bz2.BZ2File(gzipped, 'wb', compresslevel=level)
else:
raise ValueError("`compr` should be 'gz' or 'bz2'")
with open(filename, 'rb') as fil:
gz.write(fil.read())
gz.close()
if remove:
Path(filename).remove()
return gzipped
[docs]def gunzip(filename, unzipped=None, remove=True, compr='gz'):
"""Uncompress a file in gzip/bzip2 format.
Parameters:
- `filename`: compressed input file name (usually ending in '.gz' or
'.bz2')
- `unzipped`: output file name. If not specified and `filename` ends with
'.gz' or '.bz2', it will be set to the `filename` with the '.gz' or
'.bz2' removed.
If an empty string is specified or it is not specified and `filename`
does not end in '.gz' or '.bz2', the name of a temporary file is
generated. Since you will normally want to read something from the
decompressed file, this temporary file is not deleted after closing.
It is up to the user to delete it (using the returned file name) when
he is ready with it.
- `remove`: if True (default), the input file is removed after
succesful decompression. You probably want to set this to False when
decompressing to a temporary file.
- `compr`: 'gz' or 'bz2': the compression algorithm used in the input
file. If `filename` ends with either '.gz' or '.bz2', it is automatically
set from the extension. Else, the default 'gz' is used.
Returns the name of the decompressed file.
"""
filename = Path(filename)
if filename.suffix in ['.gz', '.bz2']:
compr = filename.suffix[1:]
filename = str(filename)
if compr == 'gz':
import gzip
gz = gzip.GzipFile(filename, 'rb')
elif compr == 'bz2':
import bz2
gz = bz2.BZ2File(filename, 'rb')
else:
raise ValueError("`compr` should be 'gz' or 'bz2'")
if unzipped is None and filename.endswith('.'+compr):
unzipped = filename[:-(len(compr)+1)]
if unzipped:
fil = open(unzipped, 'wb')
else:
fil = TempFile(prefix='gunzip-', delete=False)
unzipped = fil.name
fil.write(gz.read())
gz.close()
fil.close()
if remove:
Path(filename).remove
return unzipped
[docs]class File(object):
"""Transparent file compression.
This class is a context manager providing transparent file compression
and decompression. It is commonly used in a `with` statement, as follows::
with File('filename.ext','w') as f:
f.write('something')
f.write('something more')
This will create an uncompressed file with the specified name, write some
things to the file, and close it. The file can be read back similarly::
with File('filename.ext','r') as f:
for line in f:
print(f)
Because File is a context manager, the file is closed automatically when
leaving the `with` block.
By specifying a filename ending with '.gz' or '.bz2', the file will be
compressed (on writing) or decompressed (on reading) automatically.
The code can just stay the same as above.
Parameters:
- `filename`: string: name of the file to open.
If the filename ends with '.gz' or '.bz2', transparent (de)compression
will be used, with gzip or bzip2 compression algorithms respectively.
- `mode`: string: file open mode: 'r' for read, 'w' for write or 'a' for
append mode. See also the documentation for Python's open function.
For compressed files, append mode is not yet available.
- `compr`: string, one of 'gz' or 'bz2': identifies the compression
algorithm to be used: gzip or bzip2. If the file name is ending with
'.gz' or '.bz2', `compr` is set automatically from the extension.
- `level`: an integer from 1..9: gzip/bzip2 compression level.
Higher values result in smaller files, but require longer compression
times. The default of 5 gives already a fairly good compression ratio.
- `delete_temp`: bool: if True (default), the temporary files needed to
do the (de)compression are deleted when the File instance is closed.
The File class can also be used outside a `with` statement. In that case
the user has to open and close the File himself. The following are more
or less equivalent with the above examples (the `with` statement is
better at handling exceptions)::
fil = File('filename.ext','w')
f = fil.open()
f.write('something')
f.write('something more')
fil.close()
This will create an uncompressed file with the specified name, write some
things to the file, and close it. The file can be read back similarly::
fil = File('filename.ext','r')
f = fil.open()
for line in f:
print(f)
fil.close()
"""
def __init__(self, filename, mode, compr=None, level=5, delete_temp=True):
"""Initialize the File instance"""
filename = Path(filename)
if compr is None:
if filename.suffix in ['.gz', '.bz2']:
# A recognized compression format
compr = filename.suffix[1:]
self.name = filename
self.tmpfile = None
self.tmpname = None
self.mode = mode
self.compr = compr
self.level = level
self.delete = delete_temp
self.file = None
[docs] def open(self):
"""Open the File in the requested mode.
This can be used to open a File object outside a `with`
statement. It returns a Python file object that can be used
to read from or write to the File. It performs the following:
- If no compression is used, ope the file in the requested mode.
- For reading a compressed file, decompress the file to a temporary
file and open the temporary file for reading.
- For writing a compressed file, open a tem[porary file for writing.
See the documentation for the :class:`File` class for an example of
its use.
"""
if not self.compr:
# Open an uncompressed file:
# - just open the file with specified mode
self.file = open(self.name, self.mode)
elif self.mode[0:1] in 'ra':
# Open a compressed file in read or append mode:
# - first decompress file
self.tmpname = gunzip(self.name, unzipped='', remove=False)
# - then open the decompressed file in read/append mode
self.file = open(self.tmpname, self.mode)
else:
# Open a compressed file in write mode
# - open a temporary file (to be compressed after closing)
self.tmpfile = TempFile(prefix='File-', delete=False)
self.tmpname = self.tmpfile.name
self.file = self.tmpfile.file
return self.file
# this is needed to make this a context manager
__enter__ = open
[docs] def close(self):
"""Close the File.
This can be used to close the File if it was not opened using a
`with` statement. It performs the following:
- The underlying file object is closed.
- If the file was opened in write or append mode and compression is
requested, the file is compressed.
- If a temporary file was in use and delete_temp is True,
the temporary file is deleted.
See the documentation for the :class:`File` class for an example of
its use.
"""
self.file.close()
if self.compr and self.mode[0:1] in 'wa':
# - compress the resulting file
gzip(self.tmpname, gzipped=self.name, remove=True, compr=self.compr, level=self.level)
if self.tmpname and self.delete:
Path(self.tmpname).remove()
def __exit__(self, exc_type, exc_value, traceback):
"""Close the File
"""
if exc_type is None:
self.close()
return True
else:
# An exception occurred
if self.file:
self.file.close()
if self.tmpfile:
self.tmpfile.close()
if self.tmpname and self.delete:
Path(self.tmpname).remove()
return False
[docs] def reopen(self, mode='r'):
"""Reopen the file, possibly in another mode.
This allows e.g. to read back data from a just saved file
without having to destroy the File instance.
Returns the open file object.
"""
self.close()
self.mode = mode
return self.open()
# These two functions are undocumented for a reason. Believe me! BV
def splitme(s):
return s[::2], s[1::2]
def mergeme(s1, s2):
return ''.join([a+b for a, b in zip(s1, s2)])
[docs]def timeEval(s, glob=None):
"""Return the time needed for evaluating a string.
s is a string with a valid Python instructions.
The string is evaluated using Python's eval() and the difference
in seconds between the current time before and after the evaluation
is printed. The result of the evaluation is returned.
This is a simple method to measure the time spent in some operation.
It should not be used for microlevel instructions though, because
the overhead of the time calls. Use Python's timeit module to measure
microlevel execution time.
"""
start = time.time()
res = eval(s, glob)
stop = time.time()
print("Timed evaluation: %s seconds" % (stop-start))
return res
[docs]def countLines(fn):
"""Return the number of lines in a text file."""
P = system(["wc", fn])
if P.sta == 0:
return int(P.out.split()[0])
else:
return 0
##########################################################################
## Miscellaneous ##
###############################
[docs]def userName():
"""Find the name of the user."""
import getpass
return getpass.getuser()
[docs]def is_script(appname):
"""Checks whether an application name is rather a script name"""
return str(appname).endswith('.py') or str(appname).endswith('.pye')
def is_app(appname):
return not is_script(appname)
is_pyFormex = is_script
[docs]def getDocString(scriptfile):
"""Return the docstring from a script file.
This actually returns the first multiline string (delimited by
triple double quote characters) from the file.
It does relies on the script file being structured properly and
indeed including a doctring at the beginning of the file.
"""
fil = open(scriptfile, 'r')
s = fil.read()
i = s.find('"""')
if i >= 0:
j = s.find('"""', i+1)
if j >= i+3:
return s[i+3:j]
return ''
[docs]def hsorted(l):
"""Sort a list of strings in human order.
When human sort a list of strings, they tend to interprete the
numerical fields like numbers and sort these parts numerically,
instead of the lexicographic sorting by the computer.
Returns the list of strings sorted in human order.
Example:
>>> hsorted(['a1b','a11b','a1.1b','a2b','a1'])
['a1', 'a1.1b', 'a1b', 'a2b', 'a11b']
"""
def human(s):
s = RE_digits.split(s)+['0']
return list(zip(s[0::2], [int(i) for i in s[1::2]]))
return sorted(l, key=human)
[docs]def numsplit(s):
"""Split a string in numerical and non-numerical parts.
Returns a series of substrings of s. The odd items do not contain
any digits. The even items only contain digits.
Joined together, the substrings restore the original.
The number of items is always odd: if the string ends or starts with a
digit, the first or last item is an empty string.
Example:
>>> print(numsplit("aa11.22bb"))
['aa', '11', '.', '22', 'bb']
>>> print(numsplit("11.22bb"))
['', '11', '.', '22', 'bb']
>>> print(numsplit("aa11.22"))
['aa', '11', '.', '22', '']
"""
return RE_digits.split(s)
[docs]def splitDigits(s, pos=-1):
"""Split a string at a sequence of digits.
The input string is split in three parts, where the second part is
a contiguous series of digits. The second argument specifies at which
numerical substring the splitting is done. By default (pos=-1) this is
the last one.
Returns a tuple of three strings, any of which can be empty. The
second string, if non-empty is a series of digits. The first and last
items are the parts of the string before and after that series.
Any of the three return values can be an empty string.
If the string does not contain any digits, or if the specified splitting
position exceeds the number of numerical substrings, the second and
third items are empty strings.
Example:
>>> splitDigits('abc123')
('abc', '123', '')
>>> splitDigits('123')
('', '123', '')
>>> splitDigits('abc')
('abc', '', '')
>>> splitDigits('abc123def456fghi')
('abc123def', '456', 'fghi')
>>> splitDigits('abc123def456fghi',0)
('abc', '123', 'def456fghi')
>>> splitDigits('123-456')
('123-', '456', '')
>>> splitDigits('123-456',2)
('123-456', '', '')
>>> splitDigits('')
('', '', '')
"""
g = numsplit(s)
n = len(g)
i = 2*pos
if i >= -n and i+1 < n:
if i >= 0:
i += 1
#print(g,i,n)
return ''.join(g[:i]), g[i], ''.join(g[i+1:])
else:
return s, '', ''
[docs]class NameSequence(object):
"""A class for autogenerating sequences of names.
Sequences of names are autogenerated by combining a fixed part with
a numeric part. The latter is incremented at each creation of a new
name (by the next() function).
Parameters
----------
name: str
Base of the names to be generated. The name is split in three parts
(prefix, numeric, suffix), where numeric only contains digits and
suffix does not contain any digits. Thus, numeric is the last numeric
part in the string. The prefix and suffix are invariable parts, while
the numeric part will be incremented starting from the value in the
provided name.
Use ``ext`` if the variable part is not the last numeric part of name.
If ``name`` does not contain any numeric part, it is split as a file
name in stem and suffix, and '-0' is appended to the stem.
If ``name`` is empty, it will be replaced with '0'.
ext: str, optional
If provided, this is an invariable string added to the suffix from
``name`` to construct the full name template. This may contain
numeric parts, allowing the variable numeric part at any place
in the full template.
Examples
--------
>>> N = NameSequence('obj')
>>> [ next(N) for i in range(3) ]
['obj-0', 'obj-1', 'obj-2']
>>> N.peek()
'obj-3'
>>> next(N), next(N)
('obj-3', 'obj-4')
>>> N.template
'obj-%d'
>>> N = NameSequence('obj-005')
>>> [ next(N) for i in range(3) ]
['obj-005', 'obj-006', 'obj-007']
>>> N = NameSequence('abc.98')
>>> [ next(N) for i in range(3) ]
['abc.98', 'abc.99', 'abc.100']
>>> N = NameSequence('abc-8x.png')
>>> [ next(N) for i in range(3) ]
['abc-8x.png', 'abc-9x.png', 'abc-10x.png']
>>> N.template
'abc-%01dx.png'
>>> N.glob()
'abc-*x.png'
>>> next(NameSequence('abc','.png'))
'abc-0.png'
>>> next(NameSequence('abc.png'))
'abc-0.png'
>>> N = NameSequence('/home/user/abc23','5.png')
>>> [ next(N) for i in range(2) ]
['/home/user/abc235.png', '/home/user/abc245.png']
>>> N = NameSequence('')
>>> next(N), next(N)
('0', '1')
>>> N = NameSequence('12')
>>> next(N), next(N)
('12', '13')
"""
def __init__(self, name, ext=''):
"""Create a new NameSequence from name,ext."""
prefix, number, suffix = splitDigits(name)
if len(number) > 0:
self.nr = int(number)
format = "%%0%dd" % len(number)
else:
self.nr = 0
if name:
p = Path(name)
prefix, suffix = p.stem, p.suffix
format = "-%d"
else:
prefix = suffix = ''
format = "%d"
self.template = prefix+format+suffix+ext
def __next__(self):
"""Return the next name in the sequence"""
fn = self.template % self.nr
self.nr += 1
return fn
next = __next__
[docs] def peek(self):
"""Return the next name in the sequence without incrementing."""
return self.template % self.nr
[docs] def glob(self):
"""Return a UNIX glob pattern for the generated names.
A NameSequence is often used as a generator for file names.
The glob() method returns a pattern that can be used in a
UNIX-like shell command to select all the generated file names.
"""
m = re.match(r'(.*)%\d*d(.*)', self.template)
if m:
return "{}*{}".format(m.group(1), m.group(2))
else:
raise ValueError("Invalid template")
[docs]def globFiles(pattern, sort=hsorted):
"""Return a (sorted) list of files matching a filename pattern.
A function may be specified to sort/filter the list of file names.
The function should take a list of filenames as input. The output
of the function is returned. The default sort function will sort
the filenames in a human order. This will sort numeric fields in
order of increasing numbers instead of alphanumerically.
Examples
--------
>>> globFiles('pyformex/o*.py')
['pyformex/olist.py', 'pyformex/options.py']
"""
import glob
files = glob.glob(pattern)
if callable(sort):
files = sort(files)
return files
################### automatic naming of objects ##########################
_autoname = {}
[docs]def autoName(clas):
"""Return the autoname class instance for objects of type clas.
This allows for objects of a certain class to be automatically
named throughout pyFormex.
Parameters
----------
clas: str or class or object
The object class name. If a str, it is the class name. If a class,
the name is found from it. If an object, the name is taken from
the object's class. In all cases the name is converted to lower
case
Returns
-------
NameSequence instance
A NameSequence that will generate subsequent names corresponding
with the specified class.
Examples
--------
>>> from pyformex.formex import Formex
>>> F = Formex()
>>> print(next(autoName(Formex)))
formex-0
>>> print(next(autoName(F)))
formex-1
>>> print(next(autoName('Formex')))
formex-2
"""
if isinstance(clas, str):
name = clas
else:
try:
name = clas.__name__
except:
try:
name = clas.__class__.__name__
except:
raise ValueError("Expected an instance, class or string")
name = name.lower()
if not name in _autoname:
_autoname[name] = NameSequence(name+'-0')
return _autoname[name]
[docs]def prefixDict(d, prefix=''):
"""Prefix all the keys of a dict with the given prefix.
Parameters
----------
d: dict
A dict where all keys are strings.
prefix: str
A string to prepend to all keys in the dict.
Returns
-------
dict
A dict with the same contents as the input, but where all keys
have been prefixed with the given prefix string.
Examples
--------
>>> prefixDict({'a':0,'b':1},'p_')
{'p_a': 0, 'p_b': 1}
"""
return dict([(prefix+k, d[k]) for k in d])
[docs]def subDict(d, prefix='', strip=True):
"""Return a dict with the items whose key starts with prefix.
Parameters
----------
d: dict
A dict where all the keys are strings.
prefix: str
The string that is to be found at the start of the keys.
strip: bool
If True (default), the prefix is stripped from the keys.
Returns
-------
dict
A dict with all the items from ``d`` whose key starts
with ``prefix``. The keys in the returned dict will have the prefix
stripped off, unless strip=False is specified.
Examples
--------
>>> subDict({'p_a':0,'q_a':1,'p_b':2}, 'p_')
{'a': 0, 'b': 2}
>>> subDict({'p_a':0,'q_a':1,'p_b':2}, 'p_', strip=False)
{'p_a': 0, 'p_b': 2}
"""
if strip:
return dict([(k.replace(prefix, '', 1), d[k]) for k in d if k.startswith(prefix)])
else:
return dict([(k, d[k]) for k in d if k.startswith(prefix)])
[docs]def selectDict(d, keys, remove=False):
"""Return a dict with the items whose key is in keys.
Parameters
----------
d: dict
The dict to select items from.
keys: set of str
The keys to select from ``d``. This can be a set or list of key
values, or another dict, or any object having the ``key in object``
interface.
remove: bool
If True, the selected keys are removed from the input dict.
Returns
-------
dict
A dict with all the items from ``d`` whose key is in ``keys``.
See Also
--------
removeDict: the complementary operation, returns items not in ``keys``.
Examples
--------
>>> d = dict([(c,c*c) for c in range(4)])
>>> selectDict(d,[2,0])
{0: 0, 2: 4}
>>> print(d)
{0: 0, 1: 1, 2: 4, 3: 9}
>>> selectDict(d,[2,0,6],remove=True)
{0: 0, 2: 4}
>>> print(d)
{1: 1, 3: 9}
"""
keys = set(d) & set(keys)
sel = dict([(k, d[k]) for k in keys])
if remove:
for k in keys:
del d[k]
return sel
[docs]def removeDict(d, keys):
"""Return a dict with the specified keys removed.
Parameters
----------
d: dict
The dict to select items from.
keys: set of str
The keys to select from ``d``. This can be a set or list of key
values, or another dict, or any object having the ``key in object``
interface.
Returns
-------
dict
A dict with all the items from ``d`` whose key is not in ``keys``.
See Also
--------
selectDict: the complementary operation returning the items in ``keys``
Examples
--------
>>> d = dict([(c,c*c) for c in range(6)])
>>> removeDict(d,[4,0])
{1: 1, 2: 4, 3: 9, 5: 25}
"""
return dict([(k, d[k]) for k in set(d)-set(keys)])
[docs]def refreshDict(d, src):
"""Refresh a dict with values from another dict.
The values in the dict d are update with those in src.
Unlike the dict.update method, this will only update existing keys
but not add new keys.
"""
d.update(selectDict(src, d))
[docs]def inverseDict(d):
"""Return the inverse of a dictionary.
Returns a dict with keys and values interchanged.
Example:
>>> inverseDict({'a':0,'b':1})
{0: 'a', 1: 'b'}
"""
return dict([(d[k], k) for k in d])
[docs]def selectDictValues(d, values):
"""Return the keys in a dict which have a specified value
- `d`: a dict where all the keys are strings.
- `values`: a list/set of values.
The return value is a list with all the keys from d whose value
is in keys.
Example:
>>> d = dict([(c,c*c) for c in range(6)])
>>> selectDictValues(d,range(10))
[0, 1, 2, 3]
"""
return [k for k in d if d[k] in values]
[docs]def dictStr(d, skipkeys=[]):
"""Convert a dict to a string.
This is much like dict.__str__, but formats all keys as
strings and prints the items with the keys sorted.
This function is can be used as replacement for the __str__
method od dict-like classes.
A list of strings skipkeys kan be specified, to suppress the
printing of some special key values.
Example:
>>> dictStr({'a':0, 'b':1, 'c':2}, ['b'])
"{'a': 0, 'c': 2}"
"""
s = ["'%s': %r" % (k, d[k]) for k in d if k not in skipkeys]
return '{' + ', '.join(sorted(s)) + '}'
[docs]class DictDiff(object):
"""A class to compute the difference between two dictionaries
Parameters:
- `current_dict`: dict
- `past_dict`: dict
The differences are reported as sets of keys:
- items added
- items removed
- keys same in both but changed values
- keys same in both and unchanged values
"""
def __init__(self, current_dict, past_dict):
self.current_dict, self.past_dict = current_dict, past_dict
self.current_keys, self.past_keys = [
set(d.keys()) for d in (current_dict, past_dict)
]
self.intersect = self.current_keys.intersection(self.past_keys)
[docs] def added(self):
"""Return the keys in current_dict but not in past_dict"""
return self.current_keys - self.intersect
[docs] def removed(self):
"""Return the keys in past_dict but not in current_dict"""
return self.past_keys - self.intersect
[docs] def changed(self):
"""Return the keys for which the value has changed"""
return set(o for o in self.intersect
if self.past_dict[o] != self.current_dict[o])
[docs] def unchanged(self):
"""Return the keys with same value in both dicts"""
return set(o for o in self.intersect
if self.past_dict[o] == self.current_dict[o])
[docs] def equal(self):
"""Return True if both dicts are equivalent"""
return len(self.added() | self.removed() | self.changed()) == 0
def report(self):
return """Dict difference report:
(1) items added : %s
(2) items removed : %s
(3) keys same in both but changed values : %s
(4) keys same in both and unchanged values : %s
""" % (', '.join(self.added()), ', '.join(self.removed()), ', '.join(self.changed()), ', '.join(self.unchanged()))
[docs]def listAllFonts():
"""List all fonts known to the system.
Returns a sorted list of path names to all the font files
found on the system.
This uses fontconfig and will produce a warning if fontconfig is not
installed.
"""
cmd = "fc-list : file | sed 's|.*file=||;s|:||'"
P = system(cmd, shell=True)
if P.sta:
warning("fc-list could not find your font files.\nMaybe you do not have fontconfig installed?")
fonts = []
else:
fonts = sorted([Path(f.strip()) for f in P.out.split('\n')])
if len(fonts[0]) <= 1:
fonts = fonts[1:]
return fonts
[docs]def is_valid_mono_font(fontfile):
"""Filter valid monotype fonts
Parameters
----------
fontfile: Path
Path of a font file.
Returns
-------
bool
True if the provided font file has a .ttf suffix, is a fixed width
font and the font basename is not listed in the 'fonts/ignore'
configuration variable.
"""
from pyformex import freetype as ft
return fontfile.suffix == '.ttf' and \
fontfile.name not in pf.cfg['fonts/ignore'] and \
ft.Face(fontfile).is_fixed_width
[docs]def listMonoFonts():
"""List all monospace font files found on the system."""
fonts = [f for f in listAllFonts() if is_valid_mono_font(f)]
return sorted(fonts)
[docs]def defaultMonoFont():
"""Return a default monospace font for the system."""
fonts = listMonoFonts()
if not fonts:
raise ValueError("I could not find any monospace font file on your system")
for f in fonts:
if f.endswith('DejaVuSansMono.ttf'):
return f
return fonts[0]
###########################################################################
def procInfo():
print('module name: %s' % __name__)
print('parent process: %s' % os.getppid())
print('process id: %s' % os.getpid())
[docs]def interrogate(item):
"""Print useful information about item."""
from collections import OrderedDict
info = OrderedDict()
if hasattr(item, '__name__'):
info["NAME: "] = item.__name__
if hasattr(item, '__class__'):
info["CLASS: "] = item.__class__.__name__
info["ID: "] = id(item)
info["TYPE: "] = type(item)
info["VALUE: "] = repr(item)
info["CALLABLE:"] = callable(item)
if hasattr(item, '__doc__'):
doc = getattr(item, '__doc__')
doc = doc.strip() # Remove leading/trailing whitespace.
firstline = doc.split('\n')[0]
info["DOC: "] = firstline
for k in info:
print("{} {}".format(k, info[k]))
[docs]def memory_report(keys=None):
"""Return info about memory usage"""
import gc
gc.collect()
P = system('cat /proc/meminfo')
res = {}
for line in str(P.out).split('\n'):
try:
k, v = line.split(':')
k = k.strip()
v = v.replace('kB', '').strip()
res[k] = int(v)
except:
break
res['MemUsed'] = res['MemTotal'] - res['MemFree'] - res['Buffers'] - res['Cached']
if keys:
res = selectDict(res, keys)
return res
def memory_diff(mem0, mem1, tag=None):
m0 = mem0['MemUsed']/1024.
m1 = mem1['MemUsed']/1024.
m2 = m1 - m0
m3 = mem1['MemFree']/1024.
print("%10.1f MB before; %10.1f MB after; %10.1f MB used; %10.1f MB free; %s" % (m0, m1, m2, m3, tag))
_mem_state = None
def memory_track(tag=None, since=None):
global _mem_state
if since is None:
since = _mem_state
new_mem_state = memory_report()
if tag and _mem_state is not None:
memory_diff(since, new_mem_state, tag)
_mem_state = new_mem_state
return _mem_state
[docs]def totalMemSize(o, handlers={}, verbose=False):
"""Return the approximate total memory footprint of an object.
This function returns the approximate total memory footprint of an
object and all of its contents.
Automatically finds the contents of the following builtin containers and
their subclasses: tuple, list, deque, dict, set and frozenset.
To search other containers, add handlers to iterate over their contents:
handlers = {SomeContainerClass: iter,
OtherContainerClass: OtherContainerClass.get_elements}
Adapted from http://code.activestate.com/recipes/577504/
"""
from sys import getsizeof, stderr
from itertools import chain
from pyformex.collections import deque
try:
from reprlib import repr
except ImportError:
pass
def dict_handler(d):
return chain.from_iterable(d.items())
all_handlers = {
tuple: iter,
list: iter,
deque: iter,
dict: dict_handler,
set: iter,
frozenset: iter,
}
all_handlers.update(handlers) # user handlers take precedence
seen = set() # track which object id's have already been seen
default_size = getsizeof(0) # estimate sizeof object without __sizeof__
def sizeof(o):
if id(o) in seen: # do not double count the same object
return 0
seen.add(id(o))
s = getsizeof(o, default_size)
if verbose:
print(s, type(o), repr(o), file=stderr)
for typ in all_handlers:
if isinstance(o, typ):
handler = all_handlers[typ]
s += sum([sizeof(i) for i in handler(o)])
break
return s
return sizeof(o)
# THis is not a good way of computing memory usage
def vmSize():
import os
return int(os.popen('ps h o vsize %s' % os.getpid()).read().strip())
def memUsed():
return memory_report()['MemUsed']
def memory_used():
import re
rsskb = int(re.search(r'^VmRSS:\s+(\d+) kb$',
open('/proc/self/status', 'r').read(),
flags=re.IGNORECASE|re.MULTILINE).group(1))
return rsskb
###########################################################################
#### Deprecated #####
@deprecated_by('utils.runCommand', 'utils.command')
def runCommand(cmd, timeout=None, shell=True, **kargs):
P = command(cmd, timeout=timeout, shell=shell, **kargs)
return P.sta, P.out.rstrip('\n')
@deprecated_by('utils.removeTree', 'Path.removeTree')
def removeTree(path, top=True):
Path(path).removeTree(top)
@deprecated_by('utils.removeFile', 'Path.remove')
def removeFile(filename):
Path(filename).remove()
@deprecated_by('utils.mtime', 'Path.mtime')
def mtime(fn):
return os.stat(fn).st_mtime
@deprecated_by('utils.listDirs', 'Path.dirs')
def listDirs(path):
return Path(path).dirs()
@deprecated_by('utils.listFiles', 'Path.files')
def listFiles(path):
return Path(path).files()
@deprecated_by('utils.splitFilename', 'Path')
def splitFilename(path, *args, **kargs):
p = Path(path)
return p.parent, p.stem, p.suffix
@deprecated_by('utils.listTree', 'Path.listTree')
def listTree(path, *args, **kargs):
return Path(path).listTree(*args, **kargs)
@deprecated_by('utils.tempName', 'Path.TempFile')
def tempName(*args, **kargs):
return tempfile.mkstemp(*args, **kargs)[1]
@deprecated_by('utils.changeExt', 'Path.with_suffix')
def changeExt(path, ext, accept_ext=None, reject_ext=None):
p = Path(path)
oldext = p.suffix
if (accept_ext and oldext not in accept_ext or
reject_ext and oldext in reject_ext):
return str(p) + ext
else:
return str(p.with_suffix(ext))
@deprecated_by('utils.normalizeFileType', 'Path.ftype')
def normalizeFileType(ftype):
return ftype.lower().lstrip('.')
@deprecated_by('utils.fileTypeFromExt', 'Path.ftype or Path.filetype')
def fileTypeFromExt(fname):
return Path(fname).filetype()
@deprecated_by('utils.fileTypeComprFromExt', 'Path.ftype_compr')
def fileTypeComprFromExt(fname):
return Path(fname).ftype_compr()
@deprecated_by('utils.fileSize', 'Path.size')
def fileSize(fn):
return Path(fn).size
#### Removed ####
# tildeExpand = os.path.expanduser
# splitExt: replaced with fileSuffix
# listDir: was only used in listDirs & listFiles
# prefixFiles: use: [ prefix / f for f in files ]
# unPrefixFiles: use: [ f.relative_to(prefix) for f in files ]
# fileName: use: dir / (name+ext)
# fileSuffix: use Path.filetype
# currentScript
### End
