irods  4.2.8
About: iRODS (the integrated Rule Oriented Data System) is a distributed data-management system for creating data grids, digital libraries, persistent archives, and real-time data systems.
  Fossies Dox: irods-4.2.8.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

upgrade_configuration.py
Go to the documentation of this file.
1 from __future__ import print_function
2 
3 import datetime
4 import json
5 import logging
6 import os
7 import shutil
8 import stat
9 
10 from . import six
11 from . import pyparsing
12 from . import lib
13 from . import paths
14 from .exceptions import IrodsError, IrodsWarning
15 
16 def schema_version_as_int(schema_version):
17  if isinstance(schema_version, six.string_types) and schema_version[0] == 'v':
18  schema_version = schema_version[1:]
19  return int(schema_version)
20 
22  suffix = '.json'
23  if not path.endswith(suffix):
24  raise IrodsError('Configuration file %s must end with "%s"' % (path, suffix))
25  return os.path.basename(path)[:-len(suffix)]
26 
27 def requires_upgrade(irods_config):
28  if not os.path.exists(paths.version_path()):
29  return True
30  with open('.'.join([paths.version_path(), 'dist']), 'r') as f:
31  new_version = json.load(f)
32  new_version_tuple = lib.version_string_to_tuple(new_version['irods_version'])
33  old_version_tuple = lib.version_string_to_tuple(irods_config.version['irods_version'])
34  return old_version_tuple != new_version_tuple
35 
36 def upgrade(irods_config):
37  l = logging.getLogger(__name__)
38  if not os.path.exists('.'.join([paths.version_path(), 'previous'])) and os.path.exists(os.path.join(paths.irods_directory(), 'VERSION.previous')):
40 
41  with open('.'.join([paths.version_path(), 'dist']), 'r') as f:
42  new_version = json.load(f)
43  new_version['installation_time'] = datetime.datetime.now().isoformat()
44  new_version_tuple = lib.version_string_to_tuple(new_version['irods_version'])
45  if os.path.exists(paths.version_path()):
46  old_version = irods_config.version
47  old_version_tuple = lib.version_string_to_tuple(old_version['irods_version'])
48 
49  else:
50  version_previous_path = '.'.join([paths.version_path(), 'previous'])
51  if os.path.exists(version_previous_path):
52  with open(version_previous_path, 'r') as f:
53  old_version = json.load(f)
54  else :
55  old_version = {'schema_name': 'VERSION',
56  'schema_version': 'v2',
57  'irods_version': '4.1.0',
58  'commit_id': '0000000000000000000000000000000000000000',
59  'catalog_schema_version': 1,
60  'configuration_schema_version': 2}
61  old_version_tuple = lib.version_string_to_tuple(old_version['irods_version'])
62  if new_version_tuple == old_version_tuple:
63  new_version['previous_version'] = old_version
64  irods_config.commit(new_version, paths.version_path())
65 
66  if new_version_tuple == old_version_tuple:
67  l.debug('Version numbers are the same, no upgrade performed.')
68  return
69  elif new_version_tuple < old_version_tuple:
70  raise IrodsError("Downgrade detected. Downgrading is unsupported, please reinstall iRODS version %s or newer." % (old_version['irods_version']))
71  new_version['previous_version'] = old_version
72  l.debug('Upgrading from version %s to version %s.', old_version['irods_version'], new_version['irods_version'])
73 
74  if old_version_tuple < (4, 2, 0):
75  if not os.path.exists(paths.service_account_file_path()):
76  with open(paths.service_account_file_path()) as f:
77  print('IRODS_SERVICE_ACCOUNT_NAME=%s' % (irods_config.irods_user), file=f)
78  print('IRODS_SERVICE_GROUP_NAME=%s' % (irods_config.irods_group), file=f)
79 
80  old_dir_to_new_dir_map = {
81  os.path.join(paths.irods_directory(), 'iRODS', 'server', 'bin', 'cmd'): os.path.join(paths.irods_directory(), 'msiExecCmd_bin'),
82  os.path.join(paths.irods_directory(), 'iRODS', 'server', 'config', 'packedRei'): os.path.join(paths.irods_directory(), 'server', 'config', 'packedRei'),
83  os.path.join(paths.irods_directory(), 'iRODS', 'server', 'config', 'lockFileDir'): os.path.join(paths.irods_directory(), 'server', 'config', 'lockFileDir')
84  }
85  for old_dir, new_dir in old_dir_to_new_dir_map.items():
86  if os.path.isdir(old_dir):
87  for entry in os.listdir(old_dir):
88  old_path = os.path.join(old_dir, entry)
89  new_path = os.path.join(new_dir, entry)
90  if os.path.exists(new_path):
91  raise IrodsError('File conflict encountered during upgrade: %s could not be moved to %s, as %s already exists. '
92  'Please resolve this naming conflict manually and try again.' % (old_path, new_path, new_path))
93  shutil.move(old_path, new_path)
94 
95  configuration_file_list = [
96  paths.server_config_path(),
97  paths.hosts_config_path(),
98  paths.host_access_control_config_path()]
99 
100  for path in configuration_file_list:
101  upgrade_config_file(irods_config, path, new_version)
102 
103  irods_config.clear_cache()
104 
105  upgrade_config_file(irods_config, irods_config.client_environment_path, new_version, schema_name='service_account_environment')
106 
107  irods_config.commit(new_version, paths.version_path(), make_backup=True)
108 
109  os.chmod(paths.genosauth_path(),
110  stat.S_ISUID
111  | stat.S_IRUSR
112  | stat.S_IXUSR
113  | stat.S_IRGRP
114  | stat.S_IXGRP
115  | stat.S_IROTH
116  | stat.S_IXOTH
117  )
118 
119 def upgrade_config_file(irods_config, path, new_version, schema_name=None):
120  l = logging.getLogger(__name__)
121  with open(path, 'r') as f:
122  config_dict = json.load(f)
123  if schema_name is None:
124  schema_name = config_dict.get('schema_name', schema_name_from_path(path))
125 
126  if 'schema_version' in config_dict:
127  schema_version = schema_version_as_int(config_dict['schema_version'])
128  else:
129  schema_version = schema_version_as_int(new_version['previous_version']['configuration_schema_version'])
130  target_schema_version = schema_version_as_int(new_version['configuration_schema_version'])
131  if schema_version > target_schema_version:
132  raise IrodsError('Schema version (%d) in %s exceeds the schema version (%d) specified by the new version file %s.'
133  'Downgrade of configuration schemas is unsupported.' %
134  (schema_version, path, target_schema_version, '.'.join([irods_config.version_path, 'dist'])))
135  if schema_version < target_schema_version:
136  while schema_version < target_schema_version:
137  config_dict = run_schema_update(config_dict, schema_name, schema_version + 1)
138  schema_version = schema_version_as_int(config_dict['schema_version'])
139  irods_config.commit(config_dict, path, make_backup=True)
140 
141 
142 def run_schema_update(config_dict, schema_name, next_schema_version):
143  l = logging.getLogger(__name__)
144  l.debug('Upgrading %s to schema_version %d...', schema_name, next_schema_version)
145  if next_schema_version == 3:
146  config_dict['schema_name'] = schema_name
147  if schema_name == 'server_config':
148  if lib.hostname_resolves_to_local_address(config_dict['icat_host']):
149  config_dict['catalog_service_role'] = 'provider'
150  else:
151  config_dict['catalog_service_role'] = 'consumer'
152  config_dict['catalog_provider_hosts'] = [config_dict.pop('icat_host')]
153  if 'federation' in config_dict:
154  for f in config_dict['federation']:
155  f['catalog_provider_hosts'] = [f.pop('icat_host')]
156  config_dict['rule_engine_namespaces'] = [
157  ''
158  ]
159  config_dict.setdefault('plugin_configuration', {})['rule_engines'] = [
160  {
161  'instance_name': 'irods_rule_engine_plugin-irods_rule_language-instance',
162  'plugin_name': 'irods_rule_engine_plugin-irods_rule_language',
163  'plugin_specific_configuration': dict(
164  [(k, [e['filename'] for e in config_dict.pop(k)]) for k in [
165  're_data_variable_mapping_set',
166  're_function_name_mapping_set',
167  're_rulebase_set']
168  ] + [
169  ('regexes_for_supported_peps', [
170  'ac[^ ]*',
171  'msi[^ ]*',
172  '[^ ]*pep_[^ ]*_(pre|post|except|finally)'
173  ]
174  )
175  ]
176  ),
177  'shared_memory_instance': 'upgraded_irods_rule_language_rule_engine'
178  }
179  ]
180  config_dict['plugin_configuration'].setdefault('authentication', {})
181  config_dict['plugin_configuration'].setdefault('network', {})
182  config_dict['plugin_configuration'].setdefault('resource', {})
183  #put pam options in their plugin configuration
184  for k, k_prime in [
185  ('pam_no_extend', 'no_extend'),
186  ('pam_password_length', 'password_length'),
187  ('pam_password_max_time', 'password_max_time'),
188  ('pam_password_min_time', 'password_min_time')]:
189  if k in config_dict:
190  config_dict['plugin_configuration']['authentication'].setdefault('pam', {})[k_prime] = config_dict.pop(k)
191  if 'kerberos_name' in config_dict:
192  config_dict['plugin_configuration']['authentication']['krb']['name'] = config_dict.pop('kerberos_name')
193  config_dict.setdefault('advanced_settings', {})['rule_engine_server_sleep_time_in_seconds'] = 30
194  config_dict['advanced_settings']['rule_engine_server_execution_time_in_seconds'] = 120
195  if config_dict['catalog_service_role'] == 'provider':
196  with open(paths.database_config_path()) as f:
197  database_config = json.load(f)
198  config_dict['plugin_configuration'].setdefault('database', {})[database_config.pop('catalog_database_type')] = database_config
199 
200  config_dict['schema_version'] = 'v%d' % (next_schema_version)
201  return config_dict
202 
203 def load_legacy_file(filepath, load_as='dict', token=' '):
204  l = logging.getLogger(__name__)
205  l.debug('Loading %s for conversion to json...', filepath)
206  with open(filepath, 'rt') as f:
207  if load_as == 'dict':
208  container = {}
209  elif load_as == 'list':
210  container = []
211  else:
212  raise IrodsError('load_legacy_file called with unsupported load_as value: "%s".' % load_as)
213  for l in f.readlines():
214  if line and not line.startswith('#'):
215  continue
216  if load_as == 'dict':
217  key, _, val = l.strip().partition(token)
218  legacy_dict[key] = val
219  if load_as == 'list':
220  container.append(l.strip().split(token))
221  return container
222 
223 
224 #convert a legacy configuration to a v2 json configuration
226  l.debug('Attempting to convert a legacy configuration...')
227  if os.path.exists('.'.join(paths.version_path(), 'previous')):
228  l.debug('Not a legacy configuration. Skipping conversion...')
229  return
230  #load legacy files
231  legacy_server_config = load_legacy_file(os.path.join(paths.config_directory(), 'server.config'))
232  legacy_irods_config = load_legacy_file(os.path.join(paths.config_directory(), 'irods.config'))
233  legacy_irods_environment = load_legacy_file(os.getenv('irodsEnvFile',
234  os.path.join(paths.home_directory, '.irods', '.irodsEnv')))
235  legacy_irods_host = load_legacy_file(os.path.join(paths.config_directory(), 'irodsHost'), load_as='list')
236  legacy_version = load_legacy_file(os.path.join(paths.irods_directory(), 'VERSION.previous'), token='=')
237 
238  config_dicts = {}
239 
240  #Build server_config.json
241  server_config_v2 = {
242  'schema_name': 'server_config',
243  'schema_version': 'v2',
244  "advanced_settings": {
245  "default_number_of_transfer_threads": 4,
246  "default_temporary_password_lifetime_in_seconds": 120,
247  "maximum_number_of_concurrent_rule_engine_server_processes": 4,
248  "maximum_size_for_single_buffer_in_megabytes": 32,
249  "maximum_temporary_password_lifetime_in_seconds": 1000,
250  "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4,
251  "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40
252  },
253  "default_dir_mode": legacy_server_config.get('default_dir_mode', "0750"),
254  "default_file_mode": legacy_server_config.get('default_file_mode', "0600"),
255  "default_hash_scheme": legacy_server_config.get('default_hash_scheme', "SHA256"),
256  "environment_variables": {},
257  "federation": [],
258  "icat_host": legacy_server_config.get('icatHost', None),
259  "match_hash_policy": legacy_server_config.get('match_hash_policy', "compatible"),
260  "negotiation_key": legacy_server_config.get('agent_key', "TEMPORARY_32byte_negotiation_key"),
261  "re_data_variable_mapping_set": [{"filename": f} for f in legacy_server_config.get('reVariableMapSet', "core").split()],
262  "re_function_name_mapping_set": [{"filename": f} for f in legacy_server_config.get('reFuncMapSet', "core").split()],
263  "re_rulebase_set": [{"filename": f} for f in legacy_server_config.get('reRuleSet', "core").split()],
264  "schema_validation_base_uri": "file://{0}/configuration_schemas".format(paths.irods_directory()),
265  "server_control_plane_encryption_algorithm": "AES-256-CBC",
266  "server_control_plane_encryption_num_hash_rounds": 16,
267  "server_control_plane_key": "TEMPORARY__32byte_ctrl_plane_key",
268  "server_control_plane_port": 1248,
269  "server_control_plane_timeout_milliseconds": 10000,
270  "server_port_range_end": legacy_irods_config.get('$SVR_PORT_RANGE_END', 20199),
271  "server_port_range_start": legacy_irods_config.get('$SVR_PORT_RANGE_START', 20000),
272  "xmsg_port": 1279,
273  "zone_auth_scheme": "native",
274  "zone_key": legacy_server_config.get('LocalZoneSID', "TEMPORARY_zone_key"),
275  "zone_name": legacy_irods_environment.get('irodsZone', 'tempZone'),
276  "zone_port": int(legacy_irods_config.get('$IRODS_PORT', 1247)),
277  "zone_user": legacy_irods_environment.get('irodsUserName', "rods")
278  }
279 
280  #optional server config keys
281  for old_key, new_key_and_type in {
282  "KerberosName": ("kerberos_name", str),
283  "pam_password_length": ("pam_password_length", int),
284  "pam_no_extend": ("pam_no_extend", str),
285  "pam_password_min_time": ("pam_password_min_time", int),
286  "pam_password_max_time": ("pam_password_max_time", int)
287  }.items():
288  if old_key in legacy_server_config:
289  new_key, new_type = new_key_and_type
290  server_config_v2[new_key] = new_type(legacy_server_config[old_key])
291 
292  # server environment variables
293  for old_key, new_key in {
294  "spLogLevel": "sp_log_level",
295  "spLogSql": "sp_log_sql",
296  "svrPortRangeStart": "server_port_range_start",
297  "svrPortRangeEnd": "server_port_range_end",
298  "irodsEnvFile": "IRODS_ENVIRONMENT_FILE"
299  }.items():
300  if old_key in os.environ:
301  server_config_v2['environment_variables'][new_key] = os.environ[old_key]
302 
303  config_dicts[paths.server_config_path()] = server_config_v2
304 
305  if hostname_resolves_to_local_address(server_config_v2['icat_host']):
306  #Build database_config.json
307  database_config_v2 = {
308  'schema_name': 'database_config',
309  'schema_version': 'v2',
310  "catalog_database_type": legacy_server_config.get('catalog_database_type', "postgres"),
311  "db_host": legacy_irods_config.get('$DATABASE_HOST', "localhost"),
312  "db_name": legacy_irods_config.get('$DB_NAME', "ICAT"),
313  "db_odbc_type": legacy_irods_config.get('$DATABASE_ODBC_TYPE', "unix"),
314  "db_password": password_obfuscation.unscramble(legacy_server_config['DBPassword'], legacy_server_config['DBKey'], ''),
315  "db_port": int(legacy_irods_config.get('$DATABASE_PORT', 5432)),
316  "db_username": legacy_server_config.get('DBUsername', "irods")
317  }
318 
319  config_dicts[paths.database_config_path()] = database_config_v2
320 
321 
322  #Build irods_environment.json
323  irods_environment_v2 = {
324  'schema_name': 'service_account_environment',
325  'schema_version': 'v2',
326  'irods_host': legacy_irods_environment.get('irodsHost', irods.lib.get_hostname()),
327  'irods_port': int(legacy_irods_environment.get('irodsPort', server_config_v2['zone_port'])),
328  'irods_default_resource': legacy_irods_environment.get('irodsDefResource', server_config_v2['default_resource_name']),
329  'irods_home': legacy_irods_environment.get('irodsHome', '/'.join(['', server_config_v2['zone_name'], 'home', server_config_v2['zone_user']])),
330  'irods_cwd': legacy_irods_environment.get('irodsCwd', '/'.join(['', server_config_v2['zone_name'], 'home', server_config_v2['zone_user']])),
331  'irods_user_name': legacy_irods_environment.get('irodsUserName', server_config_v2['zone_user']),
332  'irods_zone_name': legacy_irods_environment.get('irodsZone', server_config_v2['zone_name']),
333  'irods_client_server_negotiation': legacy_irods_environment.get('irodsClientServerNegotiation', 'request_server_negotiation'),
334  'irods_client_server_policy': legacy_irods_environment.get('irodsClientServerPolicy', 'CS_NEG_REFUSE'),
335  'irods_encryption_key_size': int(legacy_irods_environment.get('irodsEncryptionKeySize', 32)),
336  'irods_encryption_salt_size': int(legacy_irods_environment.get('irodsEncryptionSaltSize', 8)),
337  'irods_encryption_num_hash_rounds': int(legacy_irods_environment.get('irodsEncryptionNumHashRounds', 16)),
338  'irods_encryption_algorithm': legacy_irods_environment.get('irodsEncryptionAlgorithm', 'AES-256-CBC'),
339  'irods_default_hash_scheme': legacy_irods_environment.get('irodsDefaultHashScheme', server_config_v2['default_hash_scheme']),
340  'irods_match_hash_policy': legacy_irods_environment.get('irodsMatchHashPolicy', server_config_v2['match_hash_policy']),
341  'irods_server_control_plane_port': server_config_v2['server_control_plane_port'],
342  'irods_server_control_plane_key': server_config_v2['server_control_plane_key'],
343  'irods_server_control_plane_encryption_num_hash_rounds': server_config_v2['server_control_plane_encryption_num_hash_rounds'],
344  'irods_server_control_plane_encryption_algorithm': server_config_v2['server_control_plane_encryption_algorithm'],
345  'irods_maximum_size_for_single_buffer_in_megabytes': server_config_v2['advanced_settings']['maximum_size_for_single_buffer_in_megabytes'],
346  'irods_default_number_of_transfer_threads': server_config_v2['advanced_settings']['default_number_of_transfer_threads'],
347  'irods_transfer_buffer_size_for_parallel_transfer_in_megabytes': server_config_v2['advanced_settings']['transfer_buffer_size_for_parallel_transfer_in_megabytes'],
348  "irods_ssl_verify_server": os.getenv("irodsSSLVerifyServer", 'hostname'),
349  "irods_ssl_ca_certificate_file": os.getenv("irodsSSLCACertificateFile", ''),
350  "irods_ssl_ca_certificate_path": os.getenv("irodsSSLCACertificatePath", '')
351  }
352 
353  #optional irods environment keys
354  for old_key, new_key_and_type in {
355  "xmsgHost": ("irods_xmsg_host", str),
356  "xmsgPort": ("irods_xmsg_port", int),
357  "irodsAuthScheme": ("irods_authentication_scheme", str),
358  "irodsServerDn": ("irods_gsi_server_dn", str),
359  "irodsLogLevel": ("irods_log_level", str),
360  "irodsAuthFileName": ("irods_authentication_file", str),
361  "irodsDebug": ("irods_debug", str)
362  }.items():
363  if old_key in legacy_irods_environment:
364  new_key, new_type = new_key_and_type
365  irods_environment_v2[new_key] = new_type(legacy_irods_environment[old_key])
366 
367  #optional keys from the actual environment
368  for old_key, new_key in {
369  "irodsProt": "irods_use_xml_protocol",
370  "clientUserName": "irods_client_user_name",
371  "clientRodsZone": "irods_client_zone_name",
372  }.items():
373  if old_key in os.environ:
374  server_config_v2['environment_variables'][new_key] = os.environ[old_key]
375 
376  config_dicts[server_config_v2['environment_variables'].get('IRODS_ENVIRONMENT_FILE', paths.default_client_environment_path())] = irods_environment_v2
377 
378  hosts_config_v2 = {
379  'schema_name': 'hosts_config',
380  'schema_version': 'v2',
381  'host_entries': [
382  {
383  'address_type': 'local' if l[0] == 'localhost' else 'remote',
384  'addresses': [{'address': l[i]} for i in range(0, len(l)) if i != 0 or l[i] != 'localhost']
385  } for l in legacy_irods_host]
386  }
387 
388  config_dicts[paths.hosts_config_path()] = hosts_config_v2
389 
390  host_access_control_config_v2 = {
391  'schema_name': 'host_access_control_config',
392  'schema_version': 'v2',
393  'access_entries': [{
394  'user': l[0],
395  'group': l[1],
396  'address': l[2],
397  'mask': l[3]
398  } for l in legacy_host_access_control if len(l) == 4 and l[0] != 'acChkHostAccessControl']
399  }
400 
401  config_dicts[paths.host_access_control_config_path()] = host_access_control_config_v2
402 
403  #new version file
404  previous_version = {
405  'schema_name': 'VERSION',
406  'schema_version': 'v2',
407  'irods_version': legacy_version.get('IRODSVERSION', '4.0.0'),
408  'catalog_schema_version': int(legacy_version.get('CATALOG_SCHEMA_VERSION', 1)),
409  'configuration_schema_version': 0,
410  'commit_id': '0000000000000000000000000000000000000000',
411  'build_system_information': 'unavailable',
412  'compiler_version': 'unavailable'
413  }
414 
415  for path, config_dict in config_dict.items():
416  irods_config.commit(config_dict, path, make_backup=True)
417 
418  irods_config.commit(previous_version, '.'.join(paths.version_path(), 'previous'))
irods.upgrade_configuration.upgrade_config_file
def upgrade_config_file(irods_config, path, new_version, schema_name=None)
Definition: upgrade_configuration.py:119
irods.lib.get_hostname
def get_hostname()
Definition: lib.py:35
irods.upgrade_configuration.requires_upgrade
def requires_upgrade(irods_config)
Definition: upgrade_configuration.py:27
irods.upgrade_configuration.schema_name_from_path
def schema_name_from_path(path)
Definition: upgrade_configuration.py:21
irods.pyparsing.range
range
Definition: pyparsing.py:111
irods.upgrade_configuration.schema_version_as_int
def schema_version_as_int(schema_version)
Definition: upgrade_configuration.py:16
irods.upgrade_configuration.convert_legacy_configuration_to_json
def convert_legacy_configuration_to_json(irods_config)
Definition: upgrade_configuration.py:225
hostname_resolves_to_local_address
bool hostname_resolves_to_local_address(const char *hostname)
Definition: irods_hostname.cpp:196
irods.upgrade_configuration.load_legacy_file
def load_legacy_file(filepath, load_as='dict', token=' ')
Definition: upgrade_configuration.py:203
irods.upgrade_configuration.run_schema_update
def run_schema_update(config_dict, schema_name, next_schema_version)
Definition: upgrade_configuration.py:142
int
typedef int((*funcPtr)())
irods::join
std::string join(std::vector< std::string > &strs, const std::string &separator)
Definition: irods_serialization.cpp:169
irods.upgrade_configuration.upgrade
def upgrade(irods_config)
Definition: upgrade_configuration.py:36
irods.exceptions.IrodsError
Definition: exceptions.py:1