# Part of Odoo. See LICENSE file for full copyright and licensing details. """ Modules (also called addons) management. """ from __future__ import annotations import datetime import itertools import logging import sys import time import typing import traceback import odoo.sql_db import odoo.tools.sql import odoo.tools.translate from odoo import api, tools from odoo.tools import OrderedSet from odoo.tools.convert import convert_file, IdRef, ConvertMode as LoadMode from . import db as modules_db from .migration import MigrationManager from .module import adapt_version, initialize_sys_path, load_openerp_module from .module_graph import ModuleGraph from .registry import Registry if typing.TYPE_CHECKING: from collections.abc import Collection, Iterable from odoo.api import Environment from odoo.sql_db import BaseCursor from odoo.tests.result import OdooTestResult from .module_graph import ModuleNode LoadKind = typing.Literal['data', 'demo'] _logger = logging.getLogger(__name__) def load_data(env: Environment, idref: IdRef, mode: LoadMode, kind: LoadKind, package: ModuleNode) -> bool: """ noupdate is False, unless it is demo data :returns: Whether a file was loaded """ keys = ('init_xml', 'data') if kind == 'data' else ('demo',) files: set[str] = set() for k in keys: if k == 'init_xml' and package.manifest[k]: _logger.warning("module %s: key 'init_xml' is deprecated in Odoo 19.", package.name) for filename in package.manifest[k]: if filename in files: _logger.warning("File %s is imported twice in module %s %s", filename, package.name, kind) files.add(filename) _logger.info("loading %s/%s", package.name, filename) convert_file(env, package.name, filename, idref, mode, noupdate=kind == 'demo') return bool(files) def load_demo(env: Environment, package: ModuleNode, idref: IdRef, mode: LoadMode) -> bool: """ Loads demo data for the specified package. """ try: if package.manifest.get('demo') or package.manifest.get('demo_xml'): _logger.info("Module %s: loading demo", package.name) with env.cr.savepoint(flush=False): load_data(env(su=True, context=dict(env.context, install_demo=True)), idref, mode, kind='demo', package=package) return True except Exception: # noqa: BLE001 # If we could not install demo data for this module _logger.warning( "Module %s demo data failed to install, installed without demo data", package.name, exc_info=True) todo = env.ref('base.demo_failure_todo', raise_if_not_found=False) Failure = env.get('ir.demo_failure') if todo and Failure is not None: todo.state = 'open' Failure.create({'module_id': package.id, 'error': traceback.format_exc()}) return False def force_demo(env: Environment) -> None: """ Forces the `demo` flag on all modules, and installs demo data for all installed modules. """ assert env.registry.ready env.cr.execute('UPDATE ir_module_module SET demo=True') env.cr.execute( "SELECT name FROM ir_module_module WHERE state IN ('installed', 'to upgrade', 'to remove')" ) module_list = [name for (name,) in env.cr.fetchall()] graph = ModuleGraph(env.cr, mode='load') graph.extend(module_list) for package in graph: load_demo(env, package, {}, 'init') env['ir.module.module'].invalidate_model(['demo']) # If demo data triggered module state changes (to install/upgrade/remove), # commit and rebuild registry to process button_install/upgrade calls. if env['ir.module.module'].search_count([('state', 'in', ('to install', 'to upgrade', 'to remove'))], limit=1): env.cr.commit() Registry.new(env.cr.dbname, update_module=True) env.transaction.reset() def load_module_graph( env: Environment, graph: ModuleGraph, update_module: bool = False, report: OdooTestResult | None = None, models_to_check: OrderedSet[str] | None = None, install_demo: bool = True, ) -> None: """ Load, upgrade and install not loaded module nodes in the ``graph`` for ``env.registry`` :param env: :param graph: graph of module nodes to load :param update_module: whether to update modules or not :param report: :param set models_to_check: :param install_demo: whether to attempt installing demo data for newly installed modules """ if models_to_check is None: models_to_check = OrderedSet() registry = env.registry assert isinstance(env.cr, odoo.sql_db.Cursor), "Need for a real Cursor to load modules" migrations = MigrationManager(env.cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) # register, instantiate and initialize models for each modules t0 = time.time() loading_extra_query_count = odoo.sql_db.sql_counter loading_cursor_query_count = env.cr.sql_log_count models_updated = set() for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if module_name in registry._init_modules: continue module_t0 = time.time() module_cursor_query_count = env.cr.sql_log_count module_extra_query_count = odoo.sql_db.sql_counter update_operation = ( 'install' if package.state == 'to install' else 'upgrade' if package.state == 'to upgrade' else 'reinit' if module_name in registry._reinit_modules else None ) if update_module else None module_log_level = logging.DEBUG if update_operation: module_log_level = logging.INFO _logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name, index, module_count) if update_operation: if update_operation == 'upgrade' or module_name in registry._force_upgrade_scripts: if package.name != 'base': registry._setup_models__(env.cr, []) # incremental setup migrations.migrate_module(package, 'pre') if package.name != 'base': env.flush_all() load_openerp_module(package.name) if update_operation == 'install': py_module = sys.modules['odoo.addons.%s' % (module_name,)] pre_init = package.manifest.get('pre_init_hook') if pre_init: registry._setup_models__(env.cr, []) # incremental setup getattr(py_module, pre_init)(env) model_names = registry.load(package) if update_operation: model_names = registry.descendants(model_names, '_inherit', '_inherits') models_updated |= model_names models_to_check -= model_names registry._setup_models__(env.cr, []) # incremental setup registry.init_models(env.cr, model_names, {'module': package.name}, update_operation == 'install') elif update_module and package.state != 'to remove': # The current module has simply been loaded. The models extended by this module # and for which we updated the schema, must have their schema checked again. # This is because the extension may have changed the model, # e.g. adding required=True to an existing field, but the schema has not been # updated by this module because it's not marked as 'to upgrade/to install'. model_names = registry.descendants(model_names, '_inherit', '_inherits') models_to_check |= model_names & models_updated elif update_module and package.state == 'to remove': # For all model extented (with _inherit) in the package to uninstall, we need to # update ir.model / ir.model.fields along side not-null SQL constrains. models_to_check |= model_names if update_operation: # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) module._check() idref: dict = {} if update_operation == 'install': load_data(env, idref, 'init', kind='data', package=package) if install_demo and package.demo_installable: package.demo = load_demo(env, package, idref, 'init') else: # 'upgrade' or 'reinit' # upgrading the module information module.write(module.get_values_from_terp(package.manifest)) mode = 'update' if update_operation == 'upgrade' else 'init' load_data(env, idref, mode, kind='data', package=package) if package.demo: package.demo = load_demo(env, package, idref, mode) env.cr.execute('UPDATE ir_module_module SET demo = %s WHERE id = %s', (package.demo, module_id)) module.invalidate_model(['demo']) migrations.migrate_module(package, 'post') # Update translations for all installed languages overwrite = tools.config["overwrite_existing_translations"] module._update_translations(overwrite=overwrite) if package.name is not None: registry._init_modules.add(package.name) if update_operation: if update_operation == 'install': post_init = package.manifest.get('post_init_hook') if post_init: getattr(py_module, post_init)(env) elif update_operation == 'upgrade': # validate the views that have not been checked yet env['ir.ui.view']._validate_module_views(module_name) concrete_models = [model for model in model_names if not registry[model]._abstract] if concrete_models: env.cr.execute(""" SELECT model FROM ir_model WHERE id NOT IN (SELECT DISTINCT model_id FROM ir_model_access) AND model IN %s """, [tuple(concrete_models)]) models = [model for [model] in env.cr.fetchall()] if models: lines = [ f"The models {models} have no access rules in module {module_name}, consider adding some, like:", "id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink" ] for model in models: xmlid = model.replace('.', '_') lines.append(f"{module_name}.access_{xmlid},access_{xmlid},{module_name}.model_{xmlid},base.group_user,1,0,0,0") _logger.warning('\n'.join(lines)) registry.updated_modules.append(package.name) ver = adapt_version(package.manifest['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) package.state = 'installed' module.env.flush_all() module.env.cr.commit() test_time = 0.0 test_queries = 0 test_results = None update_from_config = tools.config['update'] or tools.config['init'] or tools.config['reinit'] if tools.config['test_enable'] and (update_operation or not update_from_config): from odoo.tests import loader # noqa: PLC0415 suite = loader.make_suite([module_name], 'at_install') if suite.countTestCases(): if not update_operation: registry._setup_models__(env.cr, []) # incremental setup registry.check_null_constraints(env.cr) # Python tests tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter test_results = loader.run_suite(suite, global_report=report) assert report is not None, "Missing report during tests" report.update(test_results) test_time = time.time() - tests_t0 test_queries = odoo.sql_db.sql_counter - tests_q0 # tests may have reset the environment module = env['ir.module.module'].browse(module_id) extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries extras = [] if test_queries: extras.append(f'+{test_queries} test') if extra_queries: extras.append(f'+{extra_queries} other') _logger.log( module_log_level, "Module %s loaded in %.2fs%s, %s queries%s", module_name, time.time() - module_t0, f' (incl. {test_time:.2f}s test)' if test_time else '', env.cr.sql_log_count - module_cursor_query_count, f' ({", ".join(extras)})' if extras else '' ) if test_results and not test_results.wasSuccessful(): _logger.error( "Module %s: %d failures, %d errors of %d tests", module_name, test_results.failures_count, test_results.errors_count, test_results.testsRun ) _logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)", len(graph), time.time() - t0, env.cr.sql_log_count - loading_cursor_query_count, odoo.sql_db.sql_counter - loading_extra_query_count) # extra queries: testes, notify, any other closed cursor def _check_module_names(cr: BaseCursor, module_names: Iterable[str]) -> None: mod_names = set(module_names) mod_names.discard('all') if mod_names: cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),)) row = cr.fetchone() assert row is not None # for typing if row[0] != len(mod_names): # find out what module name(s) are incorrect: cr.execute("SELECT name FROM ir_module_module") incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) def load_modules( registry: Registry, *, update_module: bool = False, upgrade_modules: Collection[str] = (), install_modules: Collection[str] = (), reinit_modules: Collection[str] = (), new_db_demo: bool = False, models_to_check: OrderedSet[str] | None = None, ) -> None: """ Load the modules for a registry object that has just been created. This function is part of Registry.new() and should not be used anywhere else. :param registry: The new inited registry object used to load modules. :param update_module: Whether to update (install, upgrade, or uninstall) modules. Defaults to ``False`` :param upgrade_modules: A collection of module names to upgrade. :param install_modules: A collection of module names to install. :param reinit_modules: A collection of module names to reinitialize. :param new_db_demo: Whether to install demo data for new database. Defaults to ``False`` """ if models_to_check is None: models_to_check = OrderedSet() initialize_sys_path() with registry.cursor() as cr: # prevent endless wait for locks on schema changes (during online # installs) if a concurrent transaction has accessed the table; # connection settings are automatically reset when the connection is # borrowed from the pool cr.execute("SET SESSION lock_timeout = '15s'") if not modules_db.is_initialized(cr): if not update_module: _logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname) return _logger.info("Initializing database %s", cr.dbname) modules_db.initialize(cr) elif 'base' in reinit_modules: registry._reinit_modules.add('base') if 'base' in upgrade_modules: cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) graph = ModuleGraph(cr, mode='update' if update_module else 'load') graph.extend(['base']) if not graph: _logger.critical('module base cannot be loaded! (hint: verify addons-path)') raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)') if update_module and upgrade_modules: for pyfile in tools.config['pre_upgrade_scripts']: odoo.modules.migration.exec_script(cr, graph['base'].installed_version, pyfile, 'base', 'pre') if update_module and tools.sql.table_exists(cr, 'ir_model_fields'): # determine the fields which are currently translated in the database cr.execute("SELECT model || '.' || name, translate FROM ir_model_fields WHERE translate IS NOT NULL") registry._database_translated_fields = dict(cr.fetchall()) # determine the fields which are currently company dependent in the database if odoo.tools.sql.column_exists(cr, 'ir_model_fields', 'company_dependent'): cr.execute("SELECT model || '.' || name FROM ir_model_fields WHERE company_dependent IS TRUE") registry._database_company_dependent_fields = {row[0] for row in cr.fetchall()} report = registry._assertion_report env = api.Environment(cr, api.SUPERUSER_ID, {}) load_module_graph( env, graph, update_module=update_module, report=report, models_to_check=models_to_check, install_demo=new_db_demo, ) load_lang = tools.config._cli_options.pop('load_language', None) if load_lang or update_module: # some base models are used below, so make sure they are set up registry._setup_models__(cr, []) # incremental setup if load_lang: for lang in load_lang.split(','): tools.translate.load_language(cr, lang) # STEP 2: Mark other modules to be loaded/updated if update_module: Module = env['ir.module.module'] _logger.info('updating modules list') Module.update_list() _check_module_names(cr, itertools.chain(install_modules, upgrade_modules)) if install_modules: modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', tuple(install_modules))]) if modules: modules.button_install() if upgrade_modules: modules = Module.search([('state', 'in', ('installed', 'to upgrade')), ('name', 'in', tuple(upgrade_modules))]) if modules: modules.button_upgrade() if reinit_modules: modules = Module.search([('state', 'in', ('installed', 'to upgrade')), ('name', 'in', tuple(reinit_modules))]) reinit_modules = modules.downstream_dependencies(exclude_states=('uninstalled', 'uninstallable', 'to remove', 'to install')) + modules registry._reinit_modules.update(m for m in reinit_modules.mapped('name') if m not in graph._imported_modules) env.flush_all() cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) Module.invalidate_model(['state']) # STEP 3: Load marked modules (skipping base which was done in STEP 1) # loop this step in case extra modules' states are changed to 'to install'/'to update' during loading while True: if update_module: states = ('installed', 'to upgrade', 'to remove', 'to install') else: states = ('installed', 'to upgrade', 'to remove') env.cr.execute("SELECT name from ir_module_module WHERE state IN %s", [states]) module_list = [name for (name,) in env.cr.fetchall() if name not in graph] if not module_list: break graph.extend(module_list) _logger.debug('Updating graph with %d more modules', len(module_list)) updated_modules_count = len(registry.updated_modules) load_module_graph( env, graph, update_module=update_module, report=report, models_to_check=models_to_check) if len(registry.updated_modules) == updated_modules_count: break if update_module: # set up the registry without the patch for translated fields database_translated_fields = registry._database_translated_fields registry._database_translated_fields = {} registry._setup_models__(cr, []) # incremental setup # determine which translated fields should no longer be translated, # and make their model fix the database schema models_to_untranslate = set() for full_name in database_translated_fields: model_name, field_name = full_name.rsplit('.', 1) if model_name in registry: field = registry[model_name]._fields.get(field_name) if field and not field.translate: _logger.debug("Making field %s non-translated", field) models_to_untranslate.add(model_name) registry.init_models(cr, list(models_to_untranslate), {'models_to_check': True}) registry.loaded = True registry._setup_models__(cr) # check that all installed modules have been loaded by the registry Module = env['ir.module.module'] modules = Module.search_fetch(Module._get_modules_to_load_domain(), ['name'], order='name') missing = [name for name in modules.mapped('name') if name not in graph] if missing: _logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", missing) # STEP 3.5: execute migration end-scripts if update_module: migrations = MigrationManager(cr, graph) for package in graph: migrations.migrate_module(package, 'end') # check that new module dependencies have been properly installed after a migration/upgrade cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')") module_list = [name for (name,) in cr.fetchall()] if module_list: _logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list)) # STEP 3.6: apply remaining constraints in case of an upgrade registry.finalize_constraints(cr) # STEP 4: Finish and cleanup installations if registry.updated_modules: cr.execute("SELECT model from ir_model") for (model,) in cr.fetchall(): if model in registry: env[model]._check_removed_columns(log=True) elif _logger.isEnabledFor(logging.INFO): # more an info that a warning... _logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model) # Cleanup orphan records env['ir.model.data']._process_end(registry.updated_modules) # Cleanup cron vacuum_cron = env.ref('base.autovacuum_job', raise_if_not_found=False) if vacuum_cron: # trigger after a small delay to give time for assets to regenerate vacuum_cron._trigger(at=datetime.datetime.now() + datetime.timedelta(minutes=1)) env.flush_all() # STEP 5: Uninstall modules to remove if update_module: # Remove records referenced from ir_model_data for modules to be # removed (and removed the references from ir_model_data). cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',)) modules_to_remove = dict(cr.fetchall()) if modules_to_remove: pkgs = reversed([p for p in graph if p.name in modules_to_remove]) for pkg in pkgs: uninstall_hook = pkg.manifest.get('uninstall_hook') if uninstall_hook: py_module = sys.modules['odoo.addons.%s' % (pkg.name,)] getattr(py_module, uninstall_hook)(env) env.flush_all() Module = env['ir.module.module'] Module.browse(modules_to_remove.values()).module_uninstall() # Recursive reload, should only happen once, because there should be no # modules to remove next time cr.commit() _logger.info('Reloading registry once more after uninstalling modules') registry = Registry.new( cr.dbname, update_module=update_module, models_to_check=models_to_check, ) return # STEP 5.5: Verify extended fields on every model # This will fix the schema of all models in a situation such as: # - module A is loaded and defines model M; # - module B is installed/upgraded and extends model M; # - module C is loaded and extends model M; # - module B and C depend on A but not on each other; # The changes introduced by module C are not taken into account by the upgrade of B. if update_module: # We need to fix custom fields for which we have dropped the not-null constraint. cr.execute("""SELECT DISTINCT model FROM ir_model_fields WHERE state = 'manual'""") models_to_check.update(model_name for model_name, in cr.fetchall() if model_name in registry) if models_to_check: # Doesn't check models that didn't exist anymore, it might happen during uninstallation models_to_check = [model for model in models_to_check if model in registry] registry.init_models(cr, models_to_check, {'models_to_check': True, 'update_custom_fields': True}) # STEP 6: verify custom views on every model if update_module: View = env['ir.ui.view'] for model in registry: try: View._validate_custom_views(model) except Exception as e: _logger.warning('invalid custom view(s) for model %s: %s', model, e) if not registry._assertion_report or registry._assertion_report.wasSuccessful(): _logger.info('Modules loaded.') else: _logger.error('At least one test failed when loading the modules.') # STEP 9: call _register_hook on every model # This is done *exactly once* when the registry is being loaded. See the # management of those hooks in `Registry._setup_models__`: all the calls to # _setup_models__() done here do not mess up with hooks, as registry.ready # is False. for model in env.values(): model._register_hook() env.flush_all() # STEP 10: check that we can trust nullable columns registry.check_null_constraints(cr) if update_module: cr.execute( """ INSERT INTO ir_config_parameter(key, value) SELECT 'base.partially_updated_database', '1' WHERE EXISTS(SELECT FROM ir_module_module WHERE state IN ('to upgrade', 'to install', 'to remove')) ON CONFLICT DO NOTHING """ ) def reset_modules_state(db_name: str) -> None: """ Resets modules flagged as "to x" to their original state """ # Warning, this function was introduced in response to commit 763d714 # which locks cron jobs for dbs which have modules marked as 'to %'. # The goal of this function is to be called ONLY when module # installation/upgrade/uninstallation fails, which is the only known case # for which modules can stay marked as 'to %' for an indefinite amount # of time db = odoo.sql_db.db_connect(db_name) with db.cursor() as cr: if not odoo.tools.sql.table_exists(cr, 'ir_module_module'): _logger.info('skipping reset_modules_state, ir_module_module table does not exists') return cr.execute( "UPDATE ir_module_module SET state='installed' WHERE state IN ('to remove', 'to upgrade')" ) cr.execute( "UPDATE ir_module_module SET state='uninstalled' WHERE state='to install'" ) _logger.warning("Transient module states were reset")