Download: https://osdyn.ifremer.fr/pyweb/notebooks/utils/cfmg/configmanager_usage.ipynb

misc.config.ConfigManager

which allows to use some parameters stored in configuration files in a much more flexible way than what the standard module ConfigParser provides.

Create cfg files

cfgini = """
[scalars]
    boolean = boolean(default=False)
    integer = integer(default=0, min=-10, max=10)
    float = float(default=0.0, min=-10.0, max=100.0)
    string = string(default='string')
[lists]
    booleans = booleans(default=list(False, True))
    integers = integers(default=list(0, 1), min=-10, max=10)
    floats = floats(default=list(0.0), min=-10.0, max=10.0)
    strings = strings(default=list('foo', 'bar'))
""".splitlines()

# This is an example specification file called cfg.ini

# A section with scalar options
[scalars]
    boolean = boolean(default=False) # A boolean value
    integer = integer(default=0, min=-10, max=10) # An integer value between -10 and 10
    float = float(default=0.0, min=-10.0, max=10.0) # A float value between -10.0 and 10.0
    string = string(default='string') # A string value
# A section with list options
[lists]
    booleans = booleans(default=list(False, True)) # A list of booleans
    integers = integers(default=list(0, 1), min=-10, max=10) # A list of integers
    floats = floats(default=list(0.0), min=-10.0, max=10.0) # A list of floats
    strings = strings(default=list('foo', 'bar')) # A list of strings

# This is an example configuration file saved under cfg.cfg

# A section with scalar options
[scalars]
    boolean = True
    integer = 5
    float = 25.0
    string = foo
# A section with list options
[lists]
    floats = 0.0, 1.0, 1.0, 0.0
    booleans = False, True, True, False, True, False
    integers = 0, 1, 1, 0
    strings = foo, bar, bar, foo, foo bar, ' foo bar '
    test = "SaCreBleu!!"   # key which does not exist in cfg.ini
# A section which does not exists in cfg.ini
[new]
    valerie ='aiaiaie'
    bowling = True

# This is an example configuration file saved under cfgplus.cfg

# A section with scalar options
[scalars]
    boolean = True
    integer = 9
    float = 45.0
    string = foo
# A section with list options
[lists]
    floats = 0.0, 1.0, 1.0, 0.0
    booleans = True, False, True, False, True
    integers = 0, 1, 1, 0
    strings = foo, bar, bar, foo, foo bar, ' foo bar ', 'olllle'
    test = "from cfgplus.cfg"
[new]
    valerie ='aiaiaie cfgplus.cfg'
    boul = True
[more]
    youpy = cfgplus
    [[moremore]]
        fin = donecfgplus

# Get name of .ini
# __file__ = 'tutovac_optparse.ipynb'
# specfile = '%s.ini'%(P.splitext(__file__)[0])
# cfgfile = '%s.cfg'%(P.splitext(__file__)[0])

from osdyn.config import data_sample
specfile = data_sample("cfg.ini")
cfgfile = data_sample("cfg.cfg")

Different types of readings

import os, os.path as P, pprint, sys
import osdyn.utils.misc.config as C

read .ini from utils ConfigManager class

cfgm = C.ConfigManager(specfile)

cfgm

<osdyn.utils.misc.config.ConfigManager at 0x2aaad288fd30>

methods for ConfigManager class

[v for v in dir(cfgm) if not v.startswith('_')]

['arg_long_help',
 'arg_parse',
 'arg_patch',
 'cfgspecs',
 'configspecs',
 'defaults',
 'get_spec',
 'load',
 'patch',
 'reset',
 'specs',
 'validator']

ask for the defaults method to get the content of the .ini cfg file

print('\n Using the config manager\n')
cfgini = cfgm.defaults()
print('\n Default config from specfile %r:\n\n%s\n'%(specfile, pprint.pformat(cfgini.dict())))

 Using the config manager


 Default config from specfile '/home7/scratch/oasisdev/OSDYN/osdyn/data/cfg.ini':

{'lists': {'booleans': [False, True],
           'floats': [0.0],
           'integers': [0, 1],
           'strings': ['foo', 'bar']},
 'scalars': {'boolean': False, 'float': 0.0, 'integer': 0, 'string': 'string'}}

read .cfg from with class ConfigObj class

# The ConfigManager currently set defaults from its specifications when using its load method.
# So in this example we'll directly use configobj to show you only the content of the configuration file
import configobj
cfgobj = configobj.ConfigObj(cfgfile)
print('\n Content of the config file %r:\n\n%s\n'%(cfgfile, pprint.pformat(cfgobj.dict())))

 Content of the config file '/home7/scratch/oasisdev/OSDYN/osdyn/data/cfg.cfg':

{'lists': {'booleans': ['False', 'True', 'True', 'False', 'True', 'True'],
           'floats': ['0.0', '1.0', '1.0', '0.0'],
           'integers': ['0', '1', '1', '0'],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': 'True',
             'float': '25.0',
             'integer': '5',
             'string': 'foo'}}

Note

all the values are strings

read the same cfg file from ConfigManager

and see what happens

cfgc = C.ConfigManager(cfgfile)

cfgc.defaults()

ConfigObj({'scalars': {}, 'lists': {}, 'new': {}})

Warning

None specification ! You cannot validate your inputs

but you can get the content of the cfg file from the load method

# way to get the content of the configuration file (.cfg)
cfgm = C.ConfigManager(specfile)
cfgref = cfgm.load(cfgfile)
print('\n Content of the config file read from utils %r:\n\n%s\n'%(cfgfile, pprint.pformat(cfgref.dict())))

 Content of the config file read from utils '/home7/scratch/oasisdev/OSDYN/osdyn/data/cfg.cfg':

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': [0.0, 1.0, 1.0, 0.0],
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': True, 'float': 25.0, 'integer': 5, 'string': 'foo'}}

Note

Differs from cfgobj as all the value are correctly typed

But let’s see what happens if the cfg file is not correct,

because the type of value is uncorrect

cfgt = cfgm.load(data_sample('cfgbad.cfg'))

Config value error: [scalars] integer: the value "9." is of the wrong type.
Setting it to default value: 0

print('\n Content of the config file read from utils %r:\n\n%s\n'%(cfgfile, pprint.pformat(cfgt.dict())))

 Content of the config file read from utils '/home7/scratch/oasisdev/OSDYN/osdyn/data/cfg.cfg':

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': [0.0, 1.0, 1.0, 0.0],
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': True, 'float': 45.0, 'integer': 0, 'string': 'foo'}}

Warning

The integer value is returned as 0 because float is not of the correct type defined in .ini

Important

Thanks to ConfigManager and a cfg.ini file, one validates the values passed through cfg file

None validation, the values are all strings consequently

cfgnovalid = cfgm.load(cfgfile=data_sample('cfg.cfg'), validate=False)
print('\n Content of the config file, reading without validation %r:\n\n%s\n'%(cfgfile, pprint.pformat(cfgnovalid.dict())))

 Content of the config file, reading without validation '/home7/scratch/oasisdev/OSDYN/osdyn/data/cfg.cfg':

{'lists': {'booleans': ['False', 'True', 'True', 'False', 'True', 'True'],
           'floats': ['0.0', '1.0', '1.0', '0.0'],
           'integers': ['0', '1', '1', '0'],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': 'True',
             'float': '25.0',
             'integer': '5',
             'string': 'foo'}}

assert cfgnovalid == cfgobj

patch method

Patch .ini with .cfg

cfgm = C.ConfigManager(specfile)
#cfgm.patch(specfile,cfgfile) # same

cfgt = cfgm.patch(data_sample('cfg.ini'),data_sample('cfg.cfg'))
print('\n Patch .ini and .cfg :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Patch .ini and .cfg :

{'lists': {'booleans': ['False', 'True', 'True', 'False', 'True', 'True'],
           'floats': ['0.0', '1.0', '1.0', '0.0'],
           'integers': ['0', '1', '1', '0'],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': 'True',
             'float': ['2', '5', '.', '0'],
             'integer': ['5'],
             'string': 'foo'}}

Error

Mais c’est quoi cette décomposition de 25.0 ????

Warning

Patch doit combiner 2 .cfg et non pas un .ini. Donc ici, les types ne sont pas pris en compte. Pas sûre que cela explique le 2 5 . 0 mais en tout cas c’est une erreur d’utiliser pach avec un .ini

The following way to “patch” a .cfg file with a .ini is correct

cfgt = cfgm.load(cfgfile=data_sample('cfg.cfg'))
print('\n .cfg only :\n\n%s\n'%(pprint.pformat(cfgm.load(cfgfile=data_sample('cfg.cfg')).dict())))

 .cfg only :

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': [0.0, 1.0, 1.0, 0.0],
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': True, 'float': 25.0, 'integer': 5, 'string': 'foo'}}

assert cfgt == cfgref

cfgm.cfgspecs

ConfigObj({'scalars': {'boolean': 'boolean(default=False)', 'integer': 'integer(default=0, min=-10, max=10)', 'float': 'float(default=0.0, min=-10.0, max=100.0)', 'string': "string(default='string')"}, 'lists': {'booleans': 'booleans(default=list(False, True))', 'integers': 'integers(default=list(0, 1), min=-10, max=10)', 'floats': 'floats(default=list(0.0), min=-10.0, max=10.0)', 'strings': 'strings(default=list(foo,bar))'}})

#cfgm.load(cfgfile='cfg.cfg',patch=True)  # specify the name of the patch
cfgt = cfgm.load(cfgfile=data_sample('cfg.cfg'),patch=data_sample('cfgplus.cfg'))
print('\n Patch .ini and cfg.cfg + cfgplus.cfg :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Patch .ini and cfg.cfg + cfgplus.cfg :

{'lists': {'booleans': [True, False, True, False, True],
           'floats': [0.0, 1.0, 1.0, 0.0],
           'integers': [0, 1, 1, 0],
           'strings': ['foo',
                       'bar',
                       'bar',
                       'foo',
                       'foo bar',
                       ' foo bar ',
                       'olllle'],
           'test': 'from cfgplus.cfg'},
 'more': {'moremore': {'fin': 'donecfgplus'}, 'youpy': 'cfgplus'},
 'new': {'boul': 'True', 'bowling': 'True', 'valerie': 'aiaiaie cfgplus.cfg'},
 'scalars': {'boolean': True, 'float': 45.0, 'integer': 9, 'string': 'foo'}}

Note

The patch works well, even for new keys and sections. Fine !

Patch 2 .cfg without .ini

cfgm = C.ConfigManager(cfgfile)

cfgt = cfgm.patch(cfgfile,data_sample('cfgplus.cfg'))
print('\n Patch 2 .cfg without .ini :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Patch 2 .cfg without .ini :

{'lists': {'booleans': ['True', 'False', 'True', 'False', 'True'],
           'floats': ['0.0', '1.0', '1.0', '0.0'],
           'integers': ['0', '1', '1', '0'],
           'strings': ['foo',
                       'bar',
                       'bar',
                       'foo',
                       'foo bar',
                       ' foo bar ',
                       'olllle'],
           'test': 'from cfgplus.cfg'},
 'more': {'moremore': {'fin': 'donecfgplus'}, 'youpy': 'cfgplus'},
 'new': {'boul': 'True', 'bowling': 'True', 'valerie': 'aiaiaie cfgplus.cfg'},
 'scalars': {'boolean': 'True',
             'float': '45.0',
             'integer': '9',
             'string': 'foo'}}

Warning

Works but variables remain as strings

argparse method

Other way to patch .ini with .cfg

del cfgm
cfgm = C.ConfigManager(specfile)

Je ne comprends pas : cette partie passe… ou pas !

Si erreur je me retrouve avec les erreurs par défaut.

???

cfgt1 = cfgm.arg_parse(cfgfile=data_sample('cfg.cfg'), args=[], extraopts=None) # cfgfilepatch='before by default' = True
print('\n Patch .ini with .cfg file :\n\n%s\n'%(pprint.pformat(cfgt1.dict())))

 Patch .ini with .cfg file :

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': [0.0, 1.0, 1.0, 0.0],
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': False, 'float': 25.0, 'integer': 5, 'string': 'foo'}}

def check_dict(dnew, dref):
    for sec in dnew.keys():
        try:
            if dnew[sec] != dref[sec]:
                print(sec, "\n cfgnew: {}, \n cfgref : {}".format(dnew[sec].dict(), dref[sec].dict()))
        except:
            print("Error for section", sec)

check_dict(cfgt1, cfgref)

scalars
 cfgnew: {'boolean': False, 'integer': 5, 'float': 25.0, 'string': 'foo'},
 cfgref : {'boolean': True, 'integer': 5, 'float': 25.0, 'string': 'foo'}

Caution

Le booléen passe à False à cause de la ligne 1032 dans utils.misc.config:

! Feed config with command line options:
defaults.walk(_walker_argcfg_setcfg_, raise_errors=True, call_on_sections=False, cfg=cfg, options=options, nested=nested)

???

Update the specfile (.ini) with arguments passed to the script

argv = ['--scalars-string=foo,hello world', '--lists-floats=42,3.14', '--scalars-float=155.']

cfgm = C.ConfigManager(specfile)
cfi1 = cfgm.arg_parse(args=argv)
print('\n Patch .ini with arguments (already existing in .ini file) :\n\n%s\n'%(pprint.pformat(cfi1.dict())))
print('\n cfgfilepatch=before by default')

 Patch .ini with arguments (already existing in .ini file) :

{'lists': {'booleans': [False, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1],
           'strings': ['foo', 'bar']},
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 0,
             'string': 'foo,hello world'}}


 cfgfilepatch=before by default

# Make sure the differences are normal
check_dict(cfi1, cfgini)

scalars
 cfgnew: {'boolean': False, 'integer': 0, 'float': 155.0, 'string': 'foo,hello world'},
 cfgref : {'boolean': False, 'integer': 0, 'float': 0.0, 'string': 'string'}
lists
 cfgnew: {'booleans': [False, True], 'integers': [0, 1], 'floats': [42.0, 3.14], 'strings': ['foo', 'bar']},
 cfgref : {'booleans': [False, True], 'integers': [0, 1], 'floats': [0.0], 'strings': ['foo', 'bar']}

Important

When the arguments do not exist in .ini specfile, you must pass them through extraopts options of arg_parse

argv += ['--newarg=tototutu']

def split_args(argv):
    # Get the arguments passed to the script
    opts = []
    for v in argv:
        k,kv = v.split("=")
        opts.append( (k,{'default':kv}) )

    # Get the list of keys in cfg file
    keypaths = C.get_key_paths( cfgm.defaults() )

    # Keep only the arguments which are not in ConfigObj file
    extraopts = []
    for arg in opts:
        if arg[0] not in keypaths:
            extraopts.append(arg)

    return opts, extraopts

To save only the argument passed to the script

opts, extraopts = split_args(argv)

# cfi + args
cfi2 = cfgm.arg_parse(args=argv, extraopts=extraopts)
print('\n Patch .ini with arguments (one does not exist in .ini file) :\n\n%s\n'%(pprint.pformat(cfi2.dict())))
print('\n cfgfilepatch=before by default')

 Patch .ini with arguments (one does not exist in .ini file) :

{'lists': {'booleans': [False, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1],
           'strings': ['foo', 'bar']},
 'newarg': 'tototutu',
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 0,
             'string': 'foo,hello world'}}


 cfgfilepatch=before by default

# Make sure the differences are relative to newarg
check_dict(cfi2, cfi1)

Error for section newarg

Tip

Note of the use of cfgfilepatch argument to arg_parse method

    - **cfgfilepatch**: specify if the returned config must be patched if a config file command
            line option is provided and when to patch it. Can take the following values:
            - True or 'before': the config file would be used before command line options
            - 'after': the config file would be used after command line options
            - Any False like value: the config file would not be used
with
    - **cfgfile**: name of the .cfg to be patched (config.cfg by default)

Use **cfgfilepatch** and **cfgfile** together

Important

cfgfilepatch=‘before by default’ = True

# arguments only
cfi2b = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfilepatch=False)
print('\n Arguments only (.ini not patched) :\n\n%s\n'%(pprint.pformat(cfi2b.dict())))
print('\n cfgfilepatch=False')

 Arguments only (.ini not patched) :

{'lists': {'floats': [42.0, 3.14]},
 'newarg': 'tototutu',
 'scalars': {'boolean': False, 'float': 155.0, 'string': 'foo,hello world'}}


 cfgfilepatch=False

Only the arguments are introduced into the cfg object

# cfi + args
cfi2c = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfilepatch='before')
# Make sure there is no difference
check_dict(cfi2, cfi2c)

Note

42 is out of range !

- cfgfilepatch = False . No use of .ini to valid .cfg (because no .cfg file) and arguments
- cfgfilepatch = True or 'before', .cdf patched with .ini before using values passed through arguments
- cfgfilepatch = 'after', patch with .ini after using values passed through arguments

# args + .ini (because .cfg does not exists) patched (overwriten) after.
cfi2d = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfilepatch='after')
print('\n .ini (because .cfg does not exists) overwrites args :\n\n%s\n'%(pprint.pformat(cfi2d.dict())))
print('\n cfgfilepatch=after')

 .ini (because .cfg does not exists) overwrites args :

{'lists': {'booleans': [False, True],
           'floats': [0.0],
           'integers': [0, 1],
           'strings': ['foo', 'bar']},
 'newarg': 'tototutu',
 'scalars': {'boolean': False, 'float': 0.0, 'integer': 0, 'string': 'string'}}


 cfgfilepatch=after

# .ini except for additionnal key,value
check_dict(cfi2d, cfgini)

Error for section newarg

Important

Back to default values even for scalars-string. I mean : back to all the default values (except the unkown ones) because the cfgfile is not specified !!!(‘config.cfg’ by default)

also update with .cfg file

# args + .cfg, in this order
cfi2e = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfilepatch='after', cfgfile=data_sample('cfg.cfg'))
print('\n args + .cfg, in this order :\n\n%s\n'%(pprint.pformat(cfi2e.dict())))
print('\n cfgfilepatch=after + correct cfgfile')

 args + .cfg, in this order :

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': [0.0, 1.0, 1.0, 0.0],
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'newarg': 'tototutu',
 'scalars': {'boolean': True, 'float': 25.0, 'integer': 5, 'string': 'foo'}}


 cfgfilepatch=after + correct cfgfile

# Make sure the difference concerns the new key,value
check_dict(cfi2e, cfgref)

Error for section newarg

Important

As cfg.cfg is patched at the end of the process, only the new key,value (undefined in .cfg) is taken into account

cfi2f = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfile=data_sample('cfg.cfg'))
print('\n Patch .cfg with arguments (one does not exist in .ini file) :\n\n%s\n'%(pprint.pformat(cfi2f.dict())))
print('\n cfgfilepatch=before + correct cfgfile')

 Patch .cfg with arguments (one does not exist in .ini file) :

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'newarg': 'tototutu',
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 5,
             'string': 'foo,hello world'}}


 cfgfilepatch=before + correct cfgfile

# Make sure the difference concerns the arguments only
check_dict(cfi2f, cfgref)

Error for section newarg
scalars
 cfgnew: {'boolean': False, 'integer': 5, 'float': 155.0, 'string': 'foo,hello world'},
 cfgref : {'boolean': True, 'integer': 5, 'float': 25.0, 'string': 'foo'}
lists
 cfgnew: {'booleans': [False, True, True, False, True, True], 'integers': [0, 1, 1, 0], 'floats': [42.0, 3.14], 'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '], 'test': 'SaCreBleu!!'},
 cfgref : {'floats': [0.0, 1.0, 1.0, 0.0], 'booleans': [False, True, True, False, True, True], 'integers': [0, 1, 1, 0], 'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '], 'test': 'SaCreBleu!!'}

Again ‘boolean’: False !!!

Check arg_parse arguments

Tip

Argument to arg_parse method : cfgfilepatch

- **cfgfilepatch**: specify if the returned config must be patched if a config file command
        line option is provided and when to patch it. Can take the following values:
        - True or 'before': the config file would be used before command line options
        - 'after': the config file would be used after command line options
        - Any False like value: the config file would not be used

Important

cfgfilepatch=‘before by default’ = True

cfgt = cfgm.arg_parse(args=argv, extraopts=extraopts, getargs=True) # cfgfilepatch='before by default' = True
namespace = cfgt[1]  # get the passed arguments
cfgt = cfgt[0]
print('\n Patch .ini with arguments (one does not exist in .ini file) :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Patch .ini with arguments (one does not exist in .ini file) :

{'lists': {'booleans': [False, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1],
           'strings': ['foo', 'bar']},
 'newarg': 'tototutu',
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 0,
             'string': 'foo,hello world'}}

Warning

Ici je n’ai pas entré le .cfg car le nom du fichier par défaut config.cfg n’existe pas

print('\n arguments passed to the script :\n\n{}\n'.format(namespace))

 arguments passed to the script :

Namespace(newarg='tototutu', cfgfile='config.cfg', scalars_boolean=False, scalars_integer=None, scalars_float=155.0, scalars_string='foo,hello world', lists_booleans=None, lists_integers=None, lists_floats=[42.0, 3.14], lists_strings=None, vacumm_cfg=ConfigObj({'newarg': 'tototutu', 'scalars': {'boolean': False, 'integer': 0, 'float': 155.0, 'string': 'foo,hello world'}, 'lists': {'booleans': [False, True], 'integers': [0, 1], 'floats': [42.0, 3.14], 'strings': ['foo', 'bar']}}))

Tip

Argument to arg_parse method: getargs

- **getargs**: allow getting the parsed arguments in addition to the config if parse=True

Note

You get a tuple (cfg,namespace) of the patched cfg file and all the keys of the .cfg file under a Namespace format (so you get more than the arguments of the script).

difference arg_parse and patch

Tip

Argument to arg_parse method: patch

    - **cfgfilepatch**: specify if the returned config must be patched if a config file command
            line option is provided and when to patch it. Can take the following values:
            - True or 'before': the config file would be used before command line options
            - 'after': the config file would be used after command line options
            - Any False like value: the config file would not be used

with
    - **cfgfile**: name of the .cfg to be patched

- **patch**, optional: used if parse is True.
        Can take the following values:
        - a :class:`bool` value indicating wheter to apply defaults on the returned config
          before applying the command line config
        - a :class:`ConfigObj` instance to apply on the returned config
          before applying the command line config

# patch .ini cfg.cfg + arguments
cfgarg = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfile=data_sample('cfg.cfg'))

# no difference
check_dict(cfgarg, cfi2f)

# patch .ini cfg.cfg + arguments
cfgargb = cfgm.arg_parse(args=argv, extraopts=extraopts, cfgfile=cfgref)  # cfjobj

# no difference
check_dict(cfgarg, cfgargb)

Important

At this point, one patches .ini with .cfg and then with arguments passed to the script. The new arguments are taken into account (correction config.py in utils)

# patch .ini + arguments but not cfg.cfg because one must fill with a configobj
cfgt = cfgm.arg_parse(args=argv, extraopts=extraopts, patch=data_sample('cfg.cfg'))

# No difference (.ini + args)
check_dict(cfgt, cfi2c)

# patch .ini cfg.cfg (not present) + arguments
cfgt = cfgm.arg_parse(args=argv, extraopts=extraopts, patch=cfgref)
print('\n ERROR in patch .cfg with arguments (one does not exist in .ini file) :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 ERROR in patch .cfg with arguments (one does not exist in .ini file) :

{'lists': {'booleans': [False, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1],
           'strings': ['foo', 'bar'],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'newarg': 'tototutu',
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 0,
             'string': 'foo,hello world'}}

Warning

args + .cfg (only for key,value unknown in .ini) + .ini due to missing cfg.cfg because cfgfile not present (config.cfg by default so the default (.ini) is taken into account

# patch .ini cfg.cfg + arguments
cfgt2 = cfgm.arg_parse(args=argv, extraopts=extraopts, patch=cfgref, cfgfilepath=False, cfgfile=cfgref)
print('\n Patch .cfg with arguments (one does not exist in .ini file) :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Patch .cfg with arguments (one does not exist in .ini file) :

{'lists': {'booleans': [False, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1],
           'strings': ['foo', 'bar'],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'newarg': 'tototutu',
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 0,
             'string': 'foo,hello world'}}

Note

patch+cfgfilepatch=False+cfgfile and cfgfilepatch='before'[+cfgfile] have the same behabior

Update the cfg file (.cfg) with arguments passed to the script

cfgm = C.ConfigManager(cfgfile)

add arguments

argv = ['--scalars-string=foo,hello world', '--lists-floats=42,3.14', '--scalars-float=155.']

Warning

When no .ini, all the arguments are extra options. Even the ones in the .cfg file. It is the way utils.misc.config.CongifManager works, due to as call to parser.parse_args(lists(args)). You must have added extra arguments to the parser at first.

# Get the arguments passed to the script
opts = []
for v in argv:
    k,kv = v.split("=")
    opts.append( (k,{'default':kv}) )

# Keep only the arguments which are not in ConfigObj file
extraopts = []
for arg in opts:
    #if arg[0] not in keypaths:
    extraopts.append(arg)

extraopts

[('--scalars-string', {'default': 'foo,hello world'}),
 ('--lists-floats', {'default': '42,3.14'}),
 ('--scalars-float', {'default': '155.'})]

argv

['--scalars-string=foo,hello world',
 '--lists-floats=42,3.14',
 '--scalars-float=155.']

cfgref

ConfigObj({'scalars': {'boolean': True, 'integer': 5, 'float': 25.0, 'string': 'foo'}, 'lists': {'floats': [0.0, 1.0, 1.0, 0.0], 'booleans': [False, True, True, False, True, True], 'integers': [0, 1, 1, 0], 'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '], 'test': 'SaCreBleu!!'}, 'new': {'valerie': 'aiaiaie', 'bowling': 'True'}})

kwargs = {'validate':False}
cfgarg = cfgm.arg_parse(cfgfile=cfgref, args=[], extraopts=extraopts, **kwargs)
print('\n patch .cfg with arguments (no .ini) :\n\n%s\n'%(pprint.pformat(cfgarg.dict())))

 patch .cfg with arguments (no .ini) :

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': '42,3.14',
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': True,
             'float': '155.',
             'integer': 5,
             'string': 'foo,hello world'}}

# Difference in types
check_dict(cfgarg, cfi2f)

scalars
 cfgnew: {'boolean': True, 'integer': 5, 'float': '155.', 'string': 'foo,hello world'},
 cfgref : {'boolean': False, 'integer': 5, 'float': 155.0, 'string': 'foo,hello world'}
lists
 cfgnew: {'floats': '42,3.14', 'booleans': [False, True, True, False, True, True], 'integers': [0, 1, 1, 0], 'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '], 'test': 'SaCreBleu!!'},
 cfgref : {'booleans': [False, True, True, False, True, True], 'integers': [0, 1, 1, 0], 'floats': [42.0, 3.14], 'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '], 'test': 'SaCreBleu!!'}

Important

keep .ini because the arguments are not set to the correct type otherwise !

# note required kwargs = {'validate':False}
cfgarg2 = cfgm.arg_parse(cfgfile=cfgref, args=[], extraopts=extraopts)
print('\n patch .cfg with arguments (no .ini) :\n\n%s\n'%(pprint.pformat(cfgarg.dict())))

 patch .cfg with arguments (no .ini) :

{'lists': {'booleans': [False, True, True, False, True, True],
           'floats': '42,3.14',
           'integers': [0, 1, 1, 0],
           'strings': ['foo', 'bar', 'bar', 'foo', 'foo bar', ' foo bar '],
           'test': 'SaCreBleu!!'},
 'new': {'bowling': 'True', 'valerie': 'aiaiaie'},
 'scalars': {'boolean': True,
             'float': '155.',
             'integer': 5,
             'string': 'foo,hello world'}}

Config value error: [lists] booleans: the check "False, True, True, False, True, True" is unknown.
Setting it to default value: None
Config value error: [lists] floats: the check "0.0, 1.0, 1.0, 0.0" is unknown.
Setting it to default value: None
Config value error: [lists] integers: the check "0, 1, 1, 0" is unknown.
Setting it to default value: None
Config value error: [lists] strings: the check "foo, bar, bar, foo, foo bar, ' foo bar '" is unknown.
Setting it to default value: None
Config value error: [lists] test: the check "SaCreBleu!!" is unknown.
Setting it to default value: None
Config value error: [new] bowling: the check "True" is unknown.
Setting it to default value: None
Config value error: [new] valerie: the check "aiaiaie" is unknown.
Setting it to default value: None
Config value error: [scalars] boolean: the check "True" is unknown.
Setting it to default value: None
Config value error: [scalars] float: the check "25.0" is unknown.
Setting it to default value: None
Config value error: [scalars] integer: the check "5" is unknown.
Setting it to default value: None
Config value error: [scalars] string: the check "foo" is unknown.
Setting it to default value: None

Use only arguments passed to the script (nor .ini neither .cfg file)

import os, os.path as P, pprint, sys
import osdyn.utils.misc.config as C

cfgm = C.ConfigManager()

cfgm.defaults()

ConfigObj({})

argv = ['--scalars-string=foo,hello world', '--lists-floats=42,3.14', '--scalars-float=155.']

# Nor .ini neither .cfg, so all the arguments are extra options
extraopts = []
for v in argv:
    k,kv = v.split("=")
    extraopts.append( (k,{'default':kv}) )

extraopts

[('--scalars-string', {'default': 'foo,hello world'}),
 ('--lists-floats', {'default': '42,3.14'}),
 ('--scalars-float', {'default': '155.'})]

cfgt = cfgm.arg_parse(cfgfile=cfgm.defaults(), args=argv, extraopts=extraopts, cfgfilepatch=True)
print('\n Save arguments in a new cfg :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Save arguments in a new cfg :

{'lists': {'floats': '42,3.14'},
 'scalars': {'float': '155.', 'string': 'foo,hello world'}}

osdyn.utils.misc.config list_options

print('\n :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 :

{'lists': {'floats': '42,3.14'},
 'scalars': {'float': '155.', 'string': 'foo,hello world'}}

from osdyn.utils.misc.config import list_options

list_options(cfgt)

[(['scalars'], 'string'), (['scalars'], 'float'), (['lists'], 'floats')]

list_options(cfgt, sections=True) #'new')

[(['scalars'], None),
 (['scalars'], 'string'),
 (['scalars'], 'float'),
 (['lists'], None),
 (['lists'], 'floats')]

list_options(cfgt, parents=['moremore'])

[(['moremore', 'scalars'], 'string'),
 (['moremore', 'scalars'], 'float'),
 (['moremore', 'lists'], 'floats')]

list_options(cfgt, values=True)

[(['scalars'], 'string', 'foo,hello world'),
 (['scalars'], 'float', '155.'),
 (['lists'], 'floats', '42,3.14')]

list_options(cfgt, values=True, parents=['moremore'])

[(['moremore', 'scalars'], 'string', 'foo,hello world'),
 (['moremore', 'scalars'], 'float', '155.'),
 (['moremore', 'lists'], 'floats', '42,3.14')]

Create documentation from .cfg .ini file plus arguments

argv = ['--scalars-string=foo,hello world', '--lists-floats=42,3.14', '--scalars-float=155.']

cfgm = C.ConfigManager(specfile)
cfgt = cfgm.arg_parse(args=argv)
print('\n Patch .ini with arguments (already existing in .ini file) :\n\n%s\n'%(pprint.pformat(cfgt.dict())))

 Patch .ini with arguments (already existing in .ini file) :

{'lists': {'booleans': [False, True],
           'floats': [42.0, 3.14],
           'integers': [0, 1],
           'strings': ['foo', 'bar']},
 'scalars': {'boolean': False,
             'float': 155.0,
             'integer': 0,
             'string': 'foo,hello world'}}

Error

A partir de là cela foire. Il faudrait que je comprenne comment cela marche pour faire la doc mais ras le bol.

cfg = argcfg = cfgm.arg_parse(args=[‘–help’])

print(‘\n*** Command line config only:\n\n`%s:nbsphinx-math:n`’%(pprint.pformat(cfg.dict())))

cfg = argcfg = cfgm.arg_parse(args=[‘–help’])

print(‘\n*** Command line config only:\n\n`%s:nbsphinx-math:n`’%(pprint.pformat(cfg.dict())))