Dépôt officiel du code source de l'ERP OpenConcerto

/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.

 *

 * Copyright 2011 OpenConcerto, by ILM Informatique. All rights reserved.

 *

 * The contents of this file are subject to the terms of the GNU General Public License Version 3

 * only ("GPL"). You may not use this file except in compliance with the License. You can obtain a

 * copy of the License at http://www.gnu.org/licenses/gpl-3.0.html See the License for the specific

 * language governing permissions and limitations under the License.

 *

 * When distributing the software, include this License Header Notice in each file.

 */

 package org.openconcerto.erp.modules;

import org.openconcerto.erp.config.Log;

import org.openconcerto.erp.config.MainFrame;

import org.openconcerto.erp.config.MenuAndActions;

import org.openconcerto.erp.config.MenuManager;

import org.openconcerto.erp.modules.DepSolverResult.Factory;

import org.openconcerto.erp.modules.ModuleTableModel.ModuleRow;

import org.openconcerto.sql.Configuration;

import org.openconcerto.sql.TM;

import org.openconcerto.sql.element.SQLElement;

import org.openconcerto.sql.element.SQLElementDirectory;

import org.openconcerto.sql.element.SQLElementNamesFromXML;

import org.openconcerto.sql.model.AliasedTable;

import org.openconcerto.sql.model.ConnectionHandlerNoSetup;

import org.openconcerto.sql.model.DBFileCache;

import org.openconcerto.sql.model.DBItemFileCache;

import org.openconcerto.sql.model.DBRoot;

import org.openconcerto.sql.model.SQLDataSource;

import org.openconcerto.sql.model.SQLField;

import org.openconcerto.sql.model.SQLName;

import org.openconcerto.sql.model.SQLRow;

import org.openconcerto.sql.model.SQLRowListRSH;

import org.openconcerto.sql.model.SQLRowValues;

import org.openconcerto.sql.model.SQLSelect;

import org.openconcerto.sql.model.SQLSyntax;

import org.openconcerto.sql.model.SQLSystem;

import org.openconcerto.sql.model.SQLTable;

import org.openconcerto.sql.model.TableRef;

import org.openconcerto.sql.model.Where;

import org.openconcerto.sql.model.graph.DirectedEdge;

import org.openconcerto.sql.model.graph.Link.Rule;

import org.openconcerto.sql.preferences.SQLPreferences;

import org.openconcerto.sql.request.SQLFieldTranslator;

import org.openconcerto.sql.users.rights.UserRightsManager;

import org.openconcerto.sql.utils.AlterTable;

import org.openconcerto.sql.utils.ChangeTable;

import org.openconcerto.sql.utils.ChangeTable.ForeignColSpec;

import org.openconcerto.sql.utils.DropTable;

import org.openconcerto.sql.utils.SQLCreateTable;

import org.openconcerto.sql.utils.SQLUtils;

import org.openconcerto.sql.utils.SQLUtils.SQLFactory;

import org.openconcerto.sql.view.list.IListeAction;

import org.openconcerto.ui.SwingThreadUtils;

import org.openconcerto.utils.CollectionMap2.Mode;

import org.openconcerto.utils.CollectionUtils;

import org.openconcerto.utils.ExceptionHandler;

import org.openconcerto.utils.ExceptionUtils;

import org.openconcerto.utils.FileUtils;

import org.openconcerto.utils.SetMap;

import org.openconcerto.utils.StringUtils;

import org.openconcerto.utils.ThreadFactory;

import org.openconcerto.utils.Tuple2;

import org.openconcerto.utils.Tuple3;

import org.openconcerto.utils.cc.IClosure;

import org.openconcerto.utils.cc.IdentityHashSet;

import org.openconcerto.utils.cc.IdentitySet;

import org.openconcerto.utils.i18n.TranslationManager;

import org.openconcerto.xml.XMLCodecUtils;

import java.beans.XMLDecoder;

import java.beans.XMLEncoder;

import java.io.File;

import java.io.FileFilter;

import java.io.FileInputStream;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.nio.charset.StandardCharsets;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.StandardCopyOption;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collection;

import java.util.Collections;

import java.util.Date;

import java.util.HashMap;

import java.util.HashSet;

import java.util.IdentityHashMap;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedHashSet;

import java.util.LinkedList;

import java.util.List;

import java.util.ListIterator;

import java.util.Locale;

import java.util.Map;

import java.util.Map.Entry;

import java.util.ResourceBundle.Control;

import java.util.Set;

import java.util.SortedMap;

import java.util.TreeSet;

import java.util.concurrent.Callable;

import java.util.concurrent.Executor;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.FutureTask;

import java.util.concurrent.LinkedBlockingQueue;

import java.util.concurrent.ThreadPoolExecutor;

import java.util.concurrent.TimeUnit;

import java.util.logging.Level;

import java.util.logging.Logger;

import java.util.prefs.BackingStoreException;

import java.util.prefs.Preferences;

import javax.swing.SwingUtilities;

import net.jcip.annotations.GuardedBy;

import net.jcip.annotations.ThreadSafe;

/**

 * Hold the list of known modules and their status.

 *

 * @author Sylvain CUAZ

 */

@ThreadSafe

public class ModuleManager {

    /**

     * The right to install/uninstall modules in the database (everyone can install locally).

     */

    static public final String MODULE_DB_RIGHT = "moduleDBAdmin";

    static final Logger L = Logger.getLogger(ModuleManager.class.getPackage().getName());

    @GuardedBy("ModuleManager.class")

    private static ExecutorService exec = null;

    private static synchronized final Executor getExec() {

        if (exec == null)

            exec = new ThreadPoolExecutor(0, 2, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactory(ModuleManager.class.getSimpleName()

                    // not daemon since install() is not atomic

                    + " executor thread ", false));

        return exec;

    }

    static final Runnable EMPTY_RUNNABLE = new Runnable() {

        @Override

        public void run() {

        }

    };

    // public needed for XMLEncoder

    static public enum ModuleState {

        NOT_CREATED, CREATED, INSTALLED, REGISTERED, STARTED

    }

    static enum ModuleAction {

        INSTALL, START, STOP, UNINSTALL

    }

    private static final long MIN_VERSION = ModuleVersion.MIN.getMerged();

    private static final String MODULE_COLNAME = "MODULE_NAME";

    private static final String MODULE_VERSION_COLNAME = "MODULE_VERSION";

    private static final String TABLE_COLNAME = "TABLE";

    private static final String FIELD_COLNAME = "FIELD";

    private static final String ISKEY_COLNAME = "KEY";

    // Don't use String literals for the synchronized blocks

    private static final String FWK_MODULE_TABLENAME = new String("FWK_MODULE_METADATA");

    private static final String FWK_MODULE_DEP_TABLENAME = new String("FWK_MODULE_DEP");

    private static final String NEEDING_MODULE_COLNAME = "ID_MODULE";

    private static final String NEEDED_MODULE_COLNAME = "ID_MODULE_NEEDED";

    private static final String fileMutex = new String("modules");

    private static final Integer TO_INSTALL_VERSION = 1;

    private static final String OLD_BACKUP_DIR_SUFFIX = ".backup";

    private static final String OLD_FAILED_DIR_SUFFIX = ".failed";

    private static final String BACKUP_DIR = "Install backup";

    private static final String FAILED_DIR = "Install failed";

    // return true if the MainFrame is not displayable (or if there's none)

    static private boolean noDisplayableFrame() {

        final MainFrame mf = MainFrame.getInstance();

        if (mf == null)

            return true;

        final FutureTask<Boolean> f = new FutureTask<Boolean>(new Callable<Boolean>() {

            @Override

            public Boolean call() throws Exception {

                return !mf.isDisplayable();

            }

        });

        SwingThreadUtils.invoke(f);

        try {

            return f.get();

        } catch (Exception e) {

            Log.get().log(Level.WARNING, "Couldn't determine MainFrame displayability", e);

            return true;

        }

    }

    public static synchronized void tearDown() {

        if (exec != null) {

            exec.shutdown();

            exec = null;

        }

    }

    public static String getMDVariant(ModuleFactory f) {

        return getMDVariant(f.getReference());

    }

    public static String getMDVariant(ModuleReference ref) {

        return ref.getID();

    }

    static final Set<ModuleReference> versionsMapToSet(final Map<String, ModuleVersion> versions) {

        final Set<ModuleReference> res = new HashSet<ModuleReference>(versions.size());

        for (final Entry<String, ModuleVersion> e : versions.entrySet())

            res.add(new ModuleReference(e.getKey(), e.getValue()));

        return res;

    }

    // final (thus safely published) and thread-safe

    private final FactoriesByID factories;

    // to avoid starting twice the same module

    // we synchronize the whole install/start and stop/uninstall

    @GuardedBy("this")

    private final Map<String, AbstractModule> runningModules;

    // in fact it is also already guarded by "this"

    @GuardedBy("modulesElements")

    private final Map<ModuleReference, IdentityHashMap<SQLElement, SQLElement>> modulesElements;

    @GuardedBy("this")

    private boolean inited;

    // only in EDT

    private final Map<String, ComponentsContext> modulesComponents;

    // graph of created modules, ATTN since we have no way to "unload" a module we only add and

    // never remove from it

    @GuardedBy("this")

    private final DependencyGraph dependencyGraph;

    @GuardedBy("this")

    private final Map<ModuleFactory, AbstractModule> createdModules;

    // Another mutex so we can query root or conf without having to wait for modules to

    // install/uninstall, or alternatively so that start() & stop() executed in the EDT don't need

    // this monitor (see uninstallUnsafe()). This lock is a leaf lock, it mustn't call code that

    // might need another lock.

    private final Object varLock = new String("varLock");

    @GuardedBy("varLock")

    private DBRoot root;

    @GuardedBy("this")

    private SQLPreferences dbPrefs;

    @GuardedBy("varLock")

    private Configuration conf;

    @GuardedBy("varLock")

    private boolean exitAllowed;

    public ModuleManager() {

        this.factories = new FactoriesByID();

        // stopModule() needs order to reset menu

        this.runningModules = new LinkedHashMap<String, AbstractModule>();

        this.dependencyGraph = new DependencyGraph();

        this.createdModules = new LinkedHashMap<ModuleFactory, AbstractModule>();

        this.modulesElements = new HashMap<ModuleReference, IdentityHashMap<SQLElement, SQLElement>>();

        this.inited = false;

        this.modulesComponents = new HashMap<String, ComponentsContext>();

        this.root = null;

        this.dbPrefs = null;

        this.conf = null;

        this.exitAllowed = true;

    }

    /**

     * Whether the current user can manage modules.

     *

     * @return <code>true</code> if the current user can manage modules.

     */

    public final boolean currentUserIsAdmin() {

        return UserRightsManager.getCurrentUserRights().haveRight(MODULE_DB_RIGHT);

    }

    // AdminRequiredModules means installed & started

    // possible AdminForbiddenModules means neither installed nor started

    public final boolean canCurrentUser(final ModuleAction action, final ModuleRow m) {

        if (currentUserIsAdmin())

            return true;

        if (action == ModuleAction.INSTALL || action == ModuleAction.UNINSTALL)

            return canCurrentUserInstall(action, m.isInstalledRemotely());

        else if (action == ModuleAction.START)

            return true;

        else if (action == ModuleAction.STOP)

            return !m.isAdminRequired();

        else

            throw new IllegalArgumentException("Unknown action " + action);

    }

    final boolean canCurrentUserInstall(final ModuleAction action, final ModuleReference ref, final InstallationState state) {

        return this.canCurrentUserInstall(action, state.getRemote().contains(ref));

    }

    private final boolean canCurrentUserInstall(final ModuleAction action, final boolean installedRemotely) {

        if (currentUserIsAdmin())

            return true;

        if (action == ModuleAction.INSTALL)

            return installedRemotely;

        else if (action == ModuleAction.UNINSTALL)

            return !installedRemotely;

        else

            throw new IllegalArgumentException("Illegal action " + action);

    }

    // *** factories (thread-safe)

    public final int addFactories(final File dir) {

        if (!dir.exists()) {

            L.warning("Module factory directory not found: " + dir.getAbsolutePath());

            return 0;

        }

        final File[] jars = dir.listFiles(new FileFilter() {

            @Override

            public boolean accept(File f) {

                return f.getName().endsWith(".jar");

            }

        });

        int i = 0;

        if (jars != null) {

            for (final File jar : jars) {

                try {

                    this.addFactory(new JarModuleFactory(jar));

                    i++;

                } catch (Exception e) {

                    L.warning("Couldn't add " + jar);

                    e.printStackTrace();

                }

            }

        }

        return i;

    }

    public final ModuleFactory addFactoryFromPackage(File jar) throws IOException {

        final ModuleFactory f = new JarModuleFactory(jar);

        this.addFactory(f);

        return f;

    }

    /**

     * Adds a factory.

     *

     * @param f the factory to add.

     * @return the ID of the factory.

     */

    public final String addFactory(ModuleFactory f) {

        final ModuleFactory prev = this.factories.add(f);

        if (prev != null)

            L.info("Changing the factory for " + f.getReference() + "\nfrom\t" + prev + "\nto\t" + f);

        return f.getID();

    }

    public final void addFactories(Collection<ModuleFactory> factories) {

        for (final ModuleFactory f : factories)

            this.addFactory(f);

    }

    public final String addFactoryAndStart(final ModuleFactory f, final boolean persistent) {

        return this.addFactory(f, true, persistent);

    }

    private final String addFactory(final ModuleFactory f, final boolean start, final boolean persistent) {

        this.addFactory(f);

        if (start) {

            L.config("addFactory() invoked start " + (persistent ? "" : "not") + " persistent for " + f);

            this.invoke(new IClosure<ModuleManager>() {

                @Override

                public void executeChecked(ModuleManager input) {

                    try {

                        if (!startModule(f.getReference(), persistent))

                            throw new IllegalStateException("Couldn't be started");

                    } catch (Throwable e) {

                        ExceptionHandler.handle(MainFrame.getInstance(), "Unable to start " + f, e);

                    }

                }

            });

        }

        return f.getID();

    }

    public final Map<String, SortedMap<ModuleVersion, ModuleFactory>> getFactories() {

        return this.factories.getMap();

    }

    public final FactoriesByID copyFactories() {

        return new FactoriesByID(this.factories);

    }

    public final void removeFactory(String id) {

        this.factories.remove(id);

    }

    // *** modules (thread-safe)

    /**

     * Call the passed closure at a time when modules can be started. In particular this manager has

     * been set up and the {@link MainFrame#getInstance() main frame} has been created.

     *

     * @param c the closure to execute.

     */

    public void invoke(final IClosure<ModuleManager> c) {

        MainFrame.invoke(new Runnable() {

            @Override

            public void run() {

                getExec().execute(new Runnable() {

                    @Override

                    public void run() {

                        c.executeChecked(ModuleManager.this);

                    }

                });

            }

        });

    }

    /**

     * Allow to access certain methods without a full {@link #setup(DBRoot, Configuration)}. If

     * setup() is subsequently called it must be passed the same root instance.

     *

     * @param root the root.

     * @throws IllegalStateException if already set.

     * @see #getDBInstalledModules()

     * @see #getCreatedItems(String)

     */

    public final void setRoot(final DBRoot root) {

        synchronized (this.varLock) {

            if (this.root != root) {

                if (this.root != null)

                    throw new IllegalStateException("Root already set");

                this.root = root;

            }

        }

    }

    public final boolean isSetup() {

        synchronized (this.varLock) {

            return this.getRoot() != null && this.getConf() != null;

        }

    }

    /**

     * Set up the module manager.

     *

     * @param root the root where the modules install.

     * @param conf the configuration the modules change.

     * @throws IllegalStateException if already {@link #isSetup() set up}.

     */

    public final void setup(final DBRoot root, final Configuration conf) throws IllegalStateException {

        if (root == null || conf == null)

            throw new NullPointerException();

        synchronized (this.varLock) {

            if (this.isSetup())

                throw new IllegalStateException("Already setup");

            assert this.modulesElements.isEmpty() && this.runningModules.isEmpty() && this.modulesComponents.isEmpty() : "Modules cannot start without root & conf";

            this.setRoot(root);

            this.conf = conf;

        }

    }

    public synchronized final boolean isInited() {

        return this.inited;

    }

    /**

     * Initialise the module manager.

     *

     * @throws Exception if required modules couldn't be registered.

     */

    public synchronized final void init() throws Exception {

        if (!this.isSetup())

            throw new IllegalStateException("Not setup");

        // don't check this.inited, that way we could register additional elements

        SQLPreferences.getPrefTable(this.getRoot());

        final List<ModuleReference> requiredModules = new ArrayList<>(this.getAdminRequiredModules());

        final List<ModuleReference> dbRequiredModules = new ArrayList<>(this.getDBRequiredModules());

        // add modules previously chosen (before restart)

        final File toInstallFile = this.getToInstallFile();

        Set<ModuleReference> toInstall = Collections.emptySet();

        Set<ModuleReference> userReferencesToInstall = Collections.emptySet();

        boolean persistent = false;

        ModuleState toInstallTargetState = ModuleState.NOT_CREATED;

        if (toInstallFile.exists()) {

            if (!toInstallFile.canRead() || !toInstallFile.isFile()) {

                L.warning("Couldn't read " + toInstallFile);

            } else {

                try (final XMLDecoder dec = new XMLDecoder(new FileInputStream(toInstallFile))) {

                    final Number version = (Number) dec.readObject();

                    if (version.intValue() != TO_INSTALL_VERSION.intValue())

                        throw new Exception("Version mismatch, expected " + TO_INSTALL_VERSION + " found " + version);

                    final Date fileDate = (Date) dec.readObject();

                    @SuppressWarnings("unchecked")

                    final Set<ModuleReference> toInstallUnsafe = (Set<ModuleReference>) dec.readObject();

                    @SuppressWarnings("unchecked")

                    final Set<ModuleReference> userReferencesToInstallUnsafe = (Set<ModuleReference>) dec.readObject();

                    toInstallTargetState = (ModuleState) dec.readObject();

                    persistent = (Boolean) dec.readObject();

                    try {

                        final Object extra = dec.readObject();

                        assert false : "Extra object " + extra;

                    } catch (ArrayIndexOutOfBoundsException e) {

                        // OK

                    }

                    final Date now = new Date();

                    if (fileDate.compareTo(now) > 0) {

                        L.warning("File is in the future : " + fileDate);

                        // check less than 2 hours

                    } else if (now.getTime() - fileDate.getTime() > 2 * 3600 * 1000) {

                        L.warning("File is too old : " + fileDate);

                    } else {

                        // no need to check that remote and local installed haven't changed since

                        // we're using ONLY_INSTALL_ARGUMENTS

                        toInstall = toInstallUnsafe;

                        userReferencesToInstall = userReferencesToInstallUnsafe;

                        if (toInstallTargetState.compareTo(ModuleState.REGISTERED) < 0)

                            L.warning("Forcing state to " + ModuleState.REGISTERED);

                    }

                } catch (Exception e) {

                    // move out file to allow the next init() to succeed

                    final File errorFile = FileUtils.addSuffix(toInstallFile, ".error");

                    errorFile.delete();

                    final boolean renamed = toInstallFile.renameTo(errorFile);

                    throw new Exception("Couldn't parse " + toInstallFile + " ; renamed : " + renamed, e);

                } finally {

                    // Either the installation will succeed and we don't want to execute it again,

                    // or the installation will fail and we don't want to be caught in an endless

                    // loop (since there's no API to remove this file).

                    if (toInstallFile.exists() && !toInstallFile.delete())

                        throw new IOException("Couldn't delete " + toInstallFile);

                }

            }

        }

        // handle upgrades : the old version was not uninstalled and thus still in

        // dbRequiredModules, and the new version is in toInstall, so the DepSolver will error out

        // since both can't be installed at the same time.

        for (final ModuleReference toInstallRef : toInstall) {

            final Iterator<ModuleReference> iter = dbRequiredModules.iterator();

            while (iter.hasNext()) {

                final ModuleReference dbReqRef = iter.next();

                if (dbReqRef.getID().equals(toInstallRef.getID())) {

                    L.config("Ignoring DB required " + dbReqRef + " because " + toInstallRef + " was requested from the last exit");

                    iter.remove();

                }

            }

        }

        requiredModules.addAll(dbRequiredModules);

        requiredModules.addAll(toInstall);

        // if there's some choice to make, let the user make it

        final Tuple2<Solutions, ModulesStateChangeResult> modules = this.createModules(requiredModules, NoChoicePredicate.ONLY_INSTALL_ARGUMENTS, ModuleState.REGISTERED);

        if (modules.get1().getNotCreated().size() > 0)

            throw new Exception("Impossible de créer les modules, not solved : " + modules.get0().getNotSolvedReferences() + " ; not created : " + modules.get1().getNotCreated());

        if (toInstallTargetState.compareTo(ModuleState.STARTED) >= 0) {

            // make them start by startPreviouslyRunningModules() (avoiding invokeLater() of

            // createModules())

            if (persistent) {

                setPersistentModules(userReferencesToInstall);

            } else {

                // NO_CHANGE since they were just installed above

                this.createModules(userReferencesToInstall, NoChoicePredicate.NO_CHANGE, ModuleState.STARTED, persistent);

            }

        }

        this.inited = true;

    }

    // whether module removal can exit the VM

    final void setExitAllowed(boolean exitAllowed) {

        synchronized (this.varLock) {

            this.exitAllowed = exitAllowed;

        }

    }

    final boolean isExitAllowed() {

        synchronized (this.varLock) {

            return this.exitAllowed;

        }

    }

    public final boolean needExit(final ModulesStateChange solution) {

        final Set<ModuleReference> refsToRemove = solution.getReferencesToRemove();

        if (!refsToRemove.isEmpty()) {

            // only need to exit if the module is loaded into memory

            final Set<ModuleReference> registeredModules = this.getRegisteredModules();

            for (final ModuleReference toRemove : refsToRemove) {

                if (registeredModules.contains(toRemove))

                    return true;

            }

        }

        return false;

    }

    // Preferences is thread-safe

    private Preferences getPrefs() {

        // modules are installed per business entity (perhaps we could add a per user option, i.e.

        // for all businesses of all databases)

        final StringBuilder path = new StringBuilder(32);

        for (final String item : DBFileCache.getJDBCAncestorNames(getRoot(), true)) {

            path.append(StringUtils.getBoundedLengthString(DBItemFileCache.encode(item), Preferences.MAX_NAME_LENGTH));

            path.append('/');

        }

        // path must not end with '/'

        path.setLength(path.length() - 1);

        return Preferences.userNodeForPackage(ModuleManager.class).node(path.toString());

    }

    // only put references passed by the user. That way if he installs module 'A' which depends on

    // module 'util' and he later upgrade 'A' to a version which doesn't need 'util' anymore it

    // won't get started. MAYBE even offer an "auto remove" feature.

    private Preferences getRunningIDsPrefs() {

        return getPrefs().node("toRun");

    }

    protected final Preferences getDBPrefs() {

        synchronized (this) {

            if (this.dbPrefs == null) {

                final DBRoot root = getRoot();

                if (root != null)

                    this.dbPrefs = (SQLPreferences) new SQLPreferences(root).node("modules");

            }

            return this.dbPrefs;

        }

    }

    private final Preferences getRequiredIDsPrefs() {

        return getDBPrefs().node("required");

    }

    protected final boolean isModuleInstalledLocally(String id) {

        final File versionFile = getLocalVersionFile(id);

        return versionFile != null && versionFile.exists();

    }

    protected final ModuleVersion getModuleVersionInstalledLocally(String id) {

        synchronized (fileMutex) {

            final File versionFile = getLocalVersionFile(id);

            if (versionFile != null && versionFile.exists()) {

                try {

                    return new ModuleVersion(Long.valueOf(FileUtils.read(versionFile)));

                } catch (IOException e) {

                    throw new IllegalStateException("Couldn't get installed version of " + id, e);

                }

            } else {

                return null;

            }

        }

    }

    public final Set<ModuleReference> getModulesInstalledLocally() {

        return versionsMapToSet(getModulesVersionInstalledLocally());

    }

    public final Map<String, ModuleVersion> getModulesVersionInstalledLocally() {

        synchronized (fileMutex) {

            final File dir = getLocalDirectory();

            if (!dir.isDirectory())

                return Collections.emptyMap();

            final Map<String, ModuleVersion> res = new HashMap<String, ModuleVersion>();

            for (final File d : dir.listFiles()) {

                final String id = d.getName();

                final ModuleVersion version = getModuleVersionInstalledLocally(id);

                if (version != null)

                    res.put(id, version);

            }

            return res;

        }

    }

    private void setModuleInstalledLocally(ModuleReference f, boolean b) {

        try {

            synchronized (fileMutex) {

                if (b) {

                    final ModuleVersion vers = f.getVersion();

                    vers.checkValidity();

                    final File versionFile = getLocalVersionFile(f.getID());

                    FileUtils.mkdir_p(versionFile.getParentFile());

                    FileUtils.write(String.valueOf(vers.getMerged()), versionFile);

                } else {

                    // perhaps add a parameter to only remove the versionFile

                    FileUtils.rm_R(getLocalDirectory(f.getID()));

                }

            }

        } catch (IOException e) {

            throw new IllegalStateException("Couldn't change installed status of " + f, e);

        }

    }

    private SQLTable getInstalledTable(final DBRoot r) throws SQLException {

        synchronized (FWK_MODULE_TABLENAME) {

            final List<SQLCreateTable> createTables = new ArrayList<SQLCreateTable>(4);

            final SQLCreateTable createTable;

            if (!r.contains(FWK_MODULE_TABLENAME)) {

                // store :

                // - currently installed module (TABLE_COLNAME & FIELD_COLNAME are null)

                // - created tables (FIELD_COLNAME is null)

                // - created fields (and whether they are keys)

                createTable = new SQLCreateTable(r, FWK_MODULE_TABLENAME);

                createTable.setPlain(true);

                // let SQLCreateTable know which column is the primary key so that createDepTable

                // can refer to it

                createTable.addColumn(SQLSyntax.ID_NAME, createTable.getSyntax().getPrimaryIDDefinitionShort());

                createTable.setPrimaryKey(SQLSyntax.ID_NAME);

                createTable.addVarCharColumn(MODULE_COLNAME, 128);

                createTable.addColumn(TABLE_COLNAME, "varchar(128) NULL");

                createTable.addColumn(FIELD_COLNAME, "varchar(128) NULL");

                createTable.addColumn(ISKEY_COLNAME, "boolean NULL");

                createTable.addColumn(MODULE_VERSION_COLNAME, "bigint NOT NULL");

                createTable.addUniqueConstraint("uniqModule", Arrays.asList(MODULE_COLNAME, TABLE_COLNAME, FIELD_COLNAME));

                createTables.add(createTable);

            } else {

                createTable = null;

            }

            if (!r.contains(FWK_MODULE_DEP_TABLENAME)) {

                final SQLCreateTable createDepTable = new SQLCreateTable(r, FWK_MODULE_DEP_TABLENAME);

                createDepTable.setPlain(true);

                final ForeignColSpec fk, fkNeeded;

                if (createTable != null) {

                    fk = ForeignColSpec.fromCreateTable(createTable);

                    fkNeeded = ForeignColSpec.fromCreateTable(createTable);

                } else {

                    final SQLTable moduleT = r.getTable(FWK_MODULE_TABLENAME);

                    fk = ForeignColSpec.fromTable(moduleT);

                    fkNeeded = ForeignColSpec.fromTable(moduleT);

                }

                // if we remove a module, remove it dependencies

                createDepTable.addForeignColumn(fk.setColumnName(NEEDING_MODULE_COLNAME), Rule.CASCADE, Rule.CASCADE);

                // if we try to remove a module that is needed, fail

                createDepTable.addForeignColumn(fkNeeded.setColumnName(NEEDED_MODULE_COLNAME), Rule.CASCADE, Rule.RESTRICT);

                createDepTable.setPrimaryKey(NEEDING_MODULE_COLNAME, NEEDED_MODULE_COLNAME);

                createTables.add(createDepTable);

            }

            r.createTables(createTables);

        }

        return r.getTable(FWK_MODULE_TABLENAME);

    }

    private final SQLTable getDepTable() {

        return getRoot().getTable(FWK_MODULE_DEP_TABLENAME);

    }

    public final DBRoot getRoot() {

        synchronized (this.varLock) {

            return this.root;

        }

    }

    private SQLDataSource getDS() {

        return getRoot().getDBSystemRoot().getDataSource();

    }

    public final Configuration getConf() {

        synchronized (this.varLock) {

            return this.conf;

        }

    }

    private SQLElementDirectory getDirectory() {

        return getConf().getDirectory();

    }

    public final File getLocalDirectory() {

        return new File(this.getConf().getConfDir(getRoot()), "modules");

    }

    public final Set<String> migrateOldTransientDirs() {

        Set<String> res = Collections.emptySet();

        final File[] listFiles = this.getLocalDirectory().listFiles();

        if (listFiles != null) {

            for (final File f : listFiles) {

                if (f.isDirectory()) {

                    res = migrateOldDir(res, f, OLD_BACKUP_DIR_SUFFIX);

                    res = migrateOldDir(res, f, OLD_FAILED_DIR_SUFFIX);

                }

            }

        }

        return res;

    }

    private final String getOldID(final File f, final String oldSuffix) {

        if (f.getName().endsWith(oldSuffix)) {

            final String id = f.getName().substring(0, f.getName().length() - oldSuffix.length());

            return ModuleReference.checkID(id, false);

        }

        return null;

    }

    private final Set<String> migrateOldDir(Set<String> res, final File f, final String oldSuffix) {

        final String id = getOldID(f, oldSuffix);

        if (id != null) {

            final Path localFailedDir = getLocalFailedDirectory(id);

            try {

                if (Files.exists(localFailedDir)) {

                    // newer already exists, remove old one

                    FileUtils.rm_R(f);

                } else {

                    Files.createDirectories(localFailedDir.getParent());

                    Files.move(f.toPath(), localFailedDir);

                }

            } catch (IOException e) {

                L.log(Level.CONFIG, "Couldn't migrate " + f, e);

                if (res.isEmpty())

                    res = new HashSet<>();

                res.add(id);

            }

        }

        return res;

    }

    // file specifying which module (and only those, dependencies won't be installed automatically)

    // to install during the next application launch.

    private final File getToInstallFile() {

        return new File(getLocalDirectory(), "toInstall");

    }

    private final File getLocalDataSubDir(final File f) {

        // TODO return "Local Data" and a migration method

        return f;

    }

    // => create "internal state" and "local data" folders inside getLocalDirectory() (i.e.

    // module.id/local data and not local data/module.id so that we can easily copy to

    // getLocalBackupDirectory() or uninstall a module)

    private final Path getInternalStateSubDir(final Path f) {

        // TODO return "Internal State" and a migration method

        return f;

    }

    protected final File getLocalDataDirectory(final String id) {

        return getLocalDataSubDir(this.getLocalDirectory(id));

    }

    private final Path getInternalStateDirectory(final String id) {

        final File l = this.getLocalDirectory(id);

        return l == null ? null : getInternalStateSubDir(l.toPath());

    }

    private final File getLocalDirectory(final String id) {

        if (ModuleReference.checkID(id, false) == null)

            return null;

        return new File(this.getLocalDirectory(), id);

    }

    // contains a copy of local module data during install

    protected final Path getLocalBackupDirectory(final String id) {

        // will be ignored by getModulesVersionInstalledLocally()

        assert ModuleReference.checkID(BACKUP_DIR, false) == null;

        return this.getLocalDirectory().toPath().resolve(BACKUP_DIR).resolve(id);

    }

    // contains local module data of the last failed install

    protected final Path getLocalFailedDirectory(final String id) {

        // will be ignored by getModulesVersionInstalledLocally()

        assert ModuleReference.checkID(FAILED_DIR, false) == null;

        return this.getLocalDirectory().toPath().resolve(FAILED_DIR).resolve(id);

    }

    private final File getLocalVersionFile(final String id) {

        final Path dir = this.getInternalStateDirectory(id);

        if (dir == null)

            return null;

        return new File(dir.toFile(), "version");

    }

    public final ModuleVersion getDBInstalledModuleVersion(final String id) throws SQLException {

        return getDBInstalledModules(id).get(id);

    }

    public final Set<ModuleReference> getModulesInstalledRemotely() throws SQLException {

        return getDBInstalledModuleRowsByRef(null).keySet();

    }

    public final Map<String, ModuleVersion> getDBInstalledModules() throws SQLException {

        return getDBInstalledModules(null);

    }

    private final Where getModuleRowWhere(final TableRef installedTable) throws SQLException {

        return Where.isNull(installedTable.getField(TABLE_COLNAME)).and(Where.isNull(installedTable.getField(FIELD_COLNAME)));

    }

    private final List<SQLRow> getDBInstalledModuleRows(final String id) throws SQLException {

        final SQLTable installedTable = getInstalledTable(getRoot());

        final SQLSelect sel = new SQLSelect().addSelectStar(installedTable);

        sel.setWhere(getModuleRowWhere(installedTable));

        if (id != null)

            sel.andWhere(new Where(installedTable.getField(MODULE_COLNAME), "=", id));

        return SQLRowListRSH.execute(sel);

    }

    private final ModuleReference getRef(final SQLRow r) throws SQLException {

        return new ModuleReference(r.getString(MODULE_COLNAME), new ModuleVersion(r.getLong(MODULE_VERSION_COLNAME)));

    }

    private final Map<String, ModuleVersion> getDBInstalledModules(final String id) throws SQLException {

        final Map<String, ModuleVersion> res = new HashMap<String, ModuleVersion>();

        for (final SQLRow r : getDBInstalledModuleRows(id)) {

            final ModuleReference ref = getRef(r);

            res.put(ref.getID(), ref.getVersion());

        }

        return res;

    }

    private final Map<ModuleReference, SQLRow> getDBInstalledModuleRowsByRef(final String id) throws SQLException {

        final Map<ModuleReference, SQLRow> res = new HashMap<ModuleReference, SQLRow>();

        for (final SQLRow r : getDBInstalledModuleRows(id)) {

            res.put(getRef(r), r);

        }

        return res;

    }

    private SQLRow setDBInstalledModule(ModuleReference f, boolean b) throws SQLException {

        final SQLTable installedTable = getInstalledTable(getRoot());

        final Where idW = new Where(installedTable.getField(MODULE_COLNAME), "=", f.getID());

        final Where w = idW.and(getModuleRowWhere(installedTable));

        if (b) {

            final SQLSelect sel = new SQLSelect();

            sel.addSelect(installedTable.getKey());

            sel.setWhere(w);

            final Number id = (Number) installedTable.getDBSystemRoot().getDataSource().executeScalar(sel.asString());

            final SQLRowValues vals = new SQLRowValues(installedTable);

            vals.put(MODULE_VERSION_COLNAME, f.getVersion().getMerged());

            if (id != null) {

                vals.setID(id);

                return vals.update();

            } else {

                vals.put(MODULE_COLNAME, f.getID());

                vals.put(TABLE_COLNAME, null);

                vals.put(FIELD_COLNAME, null);

                return vals.insert();

            }

        } else {

            installedTable.getDBSystemRoot().getDataSource().execute("DELETE FROM " + installedTable.getSQLName().quote() + " WHERE " + w.getClause());

            installedTable.fireTableModified(SQLRow.NONEXISTANT_ID);

            return null;

        }

    }

    public final Tuple2<Set<String>, Set<SQLName>> getCreatedItems(final String id) throws SQLException {

        final SQLTable installedTable = getInstalledTable(getRoot());

        final SQLSelect sel = new SQLSelect();

        sel.addSelect(installedTable.getKey());

        sel.addSelect(installedTable.getField(TABLE_COLNAME));

        sel.addSelect(installedTable.getField(FIELD_COLNAME));

        sel.setWhere(new Where(installedTable.getField(MODULE_COLNAME), "=", id).and(Where.isNotNull(installedTable.getField(TABLE_COLNAME))));

        final Set<String> tables = new HashSet<String>();

        final Set<SQLName> fields = new HashSet<SQLName>();

        for (final SQLRow r : SQLRowListRSH.execute(sel)) {

            final String tableName = r.getString(TABLE_COLNAME);

            final String fieldName = r.getString(FIELD_COLNAME);

            if (fieldName == null)

                tables.add(tableName);

            else