Dépôt officiel du code source de l'ERP OpenConcerto

/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.

 *

 * Copyright 2011-2019 OpenConcerto, by ILM Informatique. All rights reserved.

 *

 * The contents of this file are subject to the terms of the GNU General Public License Version 3

 * only ("GPL"). You may not use this file except in compliance with the License. You can obtain a

 * copy of the License at http://www.gnu.org/licenses/gpl-3.0.html See the License for the specific

 * language governing permissions and limitations under the License.

 *

 * When distributing the software, include this License Header Notice in each file.

 */

 package org.openconcerto.sql.model;

import org.openconcerto.sql.model.SQLRowValues.CreateMode;

import org.openconcerto.sql.model.SQLRowValuesCluster.State;

import org.openconcerto.sql.model.SQLRowValuesCluster.WalkOptions;

import org.openconcerto.sql.model.graph.Link.Direction;

import org.openconcerto.sql.model.graph.Path;

import org.openconcerto.sql.model.graph.Step;

import org.openconcerto.utils.CollectionMap2Itf.ListMapItf;

import org.openconcerto.utils.CollectionUtils;

import org.openconcerto.utils.CompareUtils;

import org.openconcerto.utils.CopyUtils;

import org.openconcerto.utils.ListMap;

import org.openconcerto.utils.RTInterruptedException;

import org.openconcerto.utils.RecursionType;

import org.openconcerto.utils.SetMap;

import org.openconcerto.utils.Tuple2;

import org.openconcerto.utils.cc.ITransformer;

import org.openconcerto.utils.cc.LinkedIdentitySet;

import org.openconcerto.utils.cc.Transformer;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.Collection;

import java.util.Collections;

import java.util.Deque;

import java.util.HashMap;

import java.util.HashSet;

import java.util.LinkedHashMap;

import java.util.LinkedHashSet;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Set;

import java.util.concurrent.Callable;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Future;

import java.util.concurrent.LinkedBlockingQueue;

import java.util.concurrent.ThreadPoolExecutor;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicInteger;

import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.dbutils.ResultSetHandler;

import net.jcip.annotations.GuardedBy;

import net.jcip.annotations.ThreadSafe;

/**

 * Construct a list of linked SQLRowValues from one request.

 *

 * @author Sylvain

 */

@ThreadSafe

public class SQLRowValuesListFetcher {

    /**

     * Create an ordered fetcher with the necessary grafts to fetch the passed graph.

     *

     * @param graph what to fetch, can be any tree.

     * @return the fetcher.

     */

    public static SQLRowValuesListFetcher create(final SQLRowValues graph) {

        // ORDER shouldn't slow down the query and it makes the result predictable and repeatable

        return create(graph, true);

    }

    public static SQLRowValuesListFetcher create(final SQLRowValues graph, final boolean ordered) {

        // path -> longest referent only path

        // i.e. map each path to the main fetcher or a referent graft

        final Map<Path, Path> handledPaths = new HashMap<Path, Path>();

        final Path emptyPath = Path.get(graph.getTable());

        handledPaths.put(emptyPath, emptyPath);

        // find out referent only paths (yellow in the diagram)

        graph.getGraph().walk(graph, null, new ITransformer<State<Object>, Object>() {

            @Override

            public Path transformChecked(State<Object> input) {

                final Path p = input.getPath();

                for (int i = p.length(); i > 0; i--) {

                    final Path subPath = p.subPath(0, i);

                    if (handledPaths.containsKey(subPath))

                        break;

                    handledPaths.put(subPath, p);

                }

                return null;

            }

        }, RecursionType.DEPTH_FIRST, Direction.REFERENT);

        // find out needed grafts

        final ListMap<Path, SQLRowValuesListFetcher> grafts = new ListMap<Path, SQLRowValuesListFetcher>();

        graph.getGraph().walk(graph, null, new ITransformer<State<Object>, Object>() {

            @Override

            public Path transformChecked(State<Object> input) {

                final Path p = input.getPath();

                if (!handledPaths.containsKey(p)) {

                    final Path pMinusLast = p.minusLast();

                    if (!input.isBackwards()) {

                        // Forwards can be fetched by existing fetcher (blue in the diagram)

                        final Path existingRefPath = handledPaths.get(pMinusLast);

                        assert existingRefPath != null;

                        handledPaths.put(p, existingRefPath);

                    } else {

                        // Backwards needs another fetcher

                        if (!grafts.containsKey(pMinusLast)) {

                            final SQLRowValues copy = graph.deepCopy();

                            final SQLRowValues graftNode = copy.followPath(pMinusLast);

                            graftNode.clear();

                            final SQLRowValues previous = copy.followPath(pMinusLast.minusLast());

                            assert p.getStep(-2).isForeign();

                            previous.remove(p.getStep(-2).getSingleField().getName());

                            // don't recurse forever

                            if (previous.getGraph() == graftNode.getGraph())

                                throw new IllegalArgumentException("Graph is not a tree");

                            // ATTN pMinusLast might not be on the main fetcher so don't graft now

                            // also we can only graft non empty descendant path fetchers (plus

                            // removing a fetcher saves one request)

                            final SQLRowValuesListFetcher rec = create(graftNode, ordered);

                            final Collection<SQLRowValuesListFetcher> ungrafted = rec.ungraft();

                            if (ungrafted == null || ungrafted.size() == 0) {

                                // i.e. only one referent and thus graft not necessary

                                assert rec.descendantPath.length() > 0;

                                grafts.add(pMinusLast, rec);

                            } else {

                                grafts.addAll(pMinusLast, ungrafted);

                            }

                        }

                        throw new SQLRowValuesCluster.StopRecurseException().setCompletely(false);

                    }

                }

                return null;

            }

        }, new WalkOptions(Direction.ANY).setRecursionType(RecursionType.BREADTH_FIRST).setStartIncluded(false));

        final Set<Path> refPaths = new HashSet<Path>(handledPaths.values());

        // remove the main fetcher

        refPaths.remove(emptyPath);

        // fetchers for the referent paths (yellow part)

        final Map<Path, SQLRowValuesListFetcher> graftedFetchers;

        // create the main fetcher and grafts

        final SQLRowValuesListFetcher res;

        if (refPaths.size() == 1) {

            res = new SQLRowValuesListFetcher(graph, refPaths.iterator().next());

            graftedFetchers = Collections.emptyMap();

        } else {

            res = new SQLRowValuesListFetcher(graph, false);

            graftedFetchers = new HashMap<Path, SQLRowValuesListFetcher>();

            if (refPaths.size() > 0) {

                final Path graftPath = new Path(graph.getTable());

                final SQLRowValues copy = graph.deepCopy();

                copy.clear();

                for (final Path refPath : refPaths) {

                    final SQLRowValuesListFetcher f = new SQLRowValuesListFetcher(copy, refPath, true).setOrdered(ordered);

                    res.graft(f, graftPath);

                    graftedFetchers.put(refPath, f);

                }

            }

        }

        res.setOrdered(ordered);

        // now graft recursively created grafts

        for (final Entry<Path, ? extends Collection<SQLRowValuesListFetcher>> e : grafts.entrySet()) {

            final Path graftPath = e.getKey();

            final Path refPath = handledPaths.get(graftPath);

            // can be grafted on the main fetcher or on the referent fetchers

            final SQLRowValuesListFetcher f = graftedFetchers.containsKey(refPath) ? graftedFetchers.get(refPath) : res;

            for (final SQLRowValuesListFetcher recFetcher : e.getValue())

                f.graft(recFetcher, graftPath);

        }

        return res;

    }

    // return the referent single link path starting from graph

    private static Path computePath(SQLRowValues graph) {

        // check that there's only one referent for each row

        // (otherwise huge joins, e.g. LOCAL<-CPI,SOURCE,RECEPTEUR,etc.)

        final AtomicReference<Path> res = new AtomicReference<Path>(null);

        graph.getGraph().walk(graph, null, new ITransformer<State<Path>, Path>() {

            @Override

            public Path transformChecked(State<Path> input) {

                final Collection<SQLRowValues> referentRows = input.getCurrent().getReferentRows();

                final int size = referentRows.size();

                if (size > 1) {

                    // remove the foreign rows which are all the same (since they point to

                    // current) so the exn is more legible

                    final List<SQLRowValues> toPrint = SQLRowValues.trim(referentRows);

                    throw new IllegalArgumentException(input.getCurrent() + " is referenced by " + toPrint + "\nat " + input.getPath());

                } else if (size == 0) {

                    if (res.get() == null)

                        res.set(input.getPath());

                    else

                        throw new IllegalStateException();

                }

                return input.getAcc();

            }

        }, RecursionType.BREADTH_FIRST, Direction.REFERENT);

        // since includeStart=true

        assert res.get() != null;

        return res.get();

    }

    static private final ListMap<Tuple2<Path, Number>, SQLRowValues> createCollectionMap() {

        // we need a List in merge()

        return new ListMap<Tuple2<Path, Number>, SQLRowValues>() {

            @Override

            public List<SQLRowValues> createCollection(Collection<? extends SQLRowValues> v) {

                final List<SQLRowValues> res = new ArrayList<SQLRowValues>(8);

                res.addAll(v);

                return res;

            }

        };

    }

    // unmodifiable

    private final SQLRowValues graph;

    private final Path descendantPath;

    @GuardedBy("this")

    private List<ITransformer<SQLSelect, SQLSelect>> selTransf;

    @GuardedBy("this")

    private Number selID;

    @GuardedBy("this")

    private Set<Path> ordered;

    @GuardedBy("this")

    private boolean descendantsOrdered;

    @GuardedBy("this")

    private SQLRowValues minGraph;

    @GuardedBy("this")

    private boolean includeForeignUndef;

    @GuardedBy("this")

    private SQLSelect frozen;

    @GuardedBy("this")

    private boolean freezeRows;

    // graftPlace -> {referent path -> fetcher}, unmodifiable

    @GuardedBy("this")

    private Map<Path, Map<Path, SQLRowValuesListFetcher>> grafts;

    // {pathToAdd, existingPath}, unmodifiable

    @GuardedBy("this")

    private Map<Path, Path> postFetchLinks;

    /**

     * Construct a new instance with the passed graph of SQLRowValues.

     *

     * @param graph what SQLRowValues should be returned by {@link #fetch()}, eg <code>new

     *        SQLRowValues("SITE").setAllToNull().put("ID_CONTACT", new SQLRowValues("CONTACT"))</code>

     *        to return all sites (with all their fields) with their associated contacts.

     */

    public SQLRowValuesListFetcher(SQLRowValues graph) {

        this(graph, false);

    }

    /**

     * Construct a new instance with the passed graph of SQLRowValues. Eg if <code>graph</code> is a

     * BATIMENT which points to SITE, is pointed by LOCAL, CPI_BT and <code>referents</code> is

     * <code>true</code>, {@link #fetch()} could return

     *

     * <pre>

     * SITE[2]  BATIMENT[2]     LOCAL[2]    CPI_BT[3]

     *                                      CPI_BT[2]

     *                          LOCAL[3]

     *                          LOCAL[5]    CPI_BT[5]

     * SITE[7]  BATIMENT[3]     LOCAL[4]    CPI_BT[4]

     * SITE[7]  BATIMENT[4]

     * </pre>

     *

     * @param graph what SQLRowValues should be returned by {@link #fetch()}, eg <code>new

     *        SQLRowValues("SITE").setAllToNull().put("ID_CONTACT", new SQLRowValues("CONTACT"))</code>

     *        to return all sites (with all their fields) with their associated contacts.

     * @param referents <code>true</code> if referents to <code>graph</code> should also be fetched.

     */

    public SQLRowValuesListFetcher(SQLRowValues graph, final boolean referents) {

        this(graph, referents ? computePath(graph) : null);

    }

    /**

     * Construct a new instance.

     *

     * @param graph what SQLRowValues should be returned by {@link #fetch()}.

     * @param referentPath a {@link Path#isSingleField() single link} path from the primary table,

     *        <code>null</code> meaning don't fetch referent rows.

     */

    public SQLRowValuesListFetcher(SQLRowValues graph, final Path referentPath) {

        this(graph, referentPath, true);

    }

    /**

     * Construct a new instance.

     *

     * @param graph what SQLRowValues should be returned by {@link #fetch()}.

     * @param referentPath a {@link Path#isSingleField() single link} path from the primary table,

     *        <code>null</code> meaning don't fetch referent rows.

     * @param prune if <code>true</code> the graph will be pruned to only contain

     *        <code>referentPath</code>. If <code>false</code> the graph will be kept as is, which

     *        can produce undefined results if there exist more than one referent row outside of

     *        <code>referentPath</code>.

     */

    SQLRowValuesListFetcher(final SQLRowValues graph, final Path referentPath, final boolean prune) {

        super();

        this.graph = graph.deepCopy();

        this.descendantPath = referentPath == null ? Path.get(graph.getTable()) : referentPath;

        if (!this.descendantPath.isDirection(Direction.REFERENT))

            throw new IllegalArgumentException("path is not (exclusively) referent : " + this.descendantPath);

        final SQLRowValues descRow = this.graph.followPath(this.descendantPath);

        if (descRow == null)

            throw new IllegalArgumentException("path is not contained in the passed rowValues : " + referentPath + "\n" + this.graph.printTree());

        // followPath() do the following check

        assert this.descendantPath.getFirst() == this.graph.getTable() && this.descendantPath.isSingleField();

        if (prune) {

            this.graph.getGraph().walk(descRow, null, new ITransformer<State<Object>, Object>() {

                @Override

                public Object transformChecked(State<Object> input) {

                    if (input.getFrom() == null) {

                        input.getCurrent().clearReferents();

                    } else {

                        input.getCurrent().retainReferent(input.getPrevious());

                    }

                    return null;

                }

            }, RecursionType.BREADTH_FIRST, Direction.FOREIGN);

        }

        // always need IDs

        for (final SQLRowValues curr : this.graph.getGraph().getItems()) {

            // don't overwrite existing values

            if (!curr.hasID())

                curr.setID(null);

        }

        this.graph.getGraph().freeze();

        synchronized (this) {

            this.selTransf = Collections.emptyList();

            this.selID = null;

            this.ordered = Collections.<Path> emptySet();

            this.descendantsOrdered = false;

            this.minGraph = null;

            this.includeForeignUndef = false;

            this.frozen = null;

            this.freezeRows = false;

            this.grafts = Collections.emptyMap();

            this.postFetchLinks = Collections.emptyMap();

        }

    }

    // be aware that the new instance will share the same selTransf, and if it doesn't directly

    // (with copyTransf) some state can still be shared

    private SQLRowValuesListFetcher(SQLRowValuesListFetcher f, final boolean copyTransf) {

        synchronized (f) {

            this.graph = f.getGraph().toImmutable();

            this.descendantPath = f.getReferentPath();

            // can't deadlock since this hasn't been published

            synchronized (this) {

                this.selTransf = copyTransf ? CopyUtils.copy(f.selTransf) : f.selTransf;

                this.selID = f.getSelID();

                this.ordered = f.getOrder();

                this.descendantsOrdered = f.areReferentsOrdered();

                this.minGraph = f.minGraph == null ? null : f.minGraph.toImmutable();

                this.includeForeignUndef = f.includeForeignUndef;

                // a new instance is always mutable

                this.frozen = null;

                this.freezeRows = f.freezeRows;

                // Recursively copy grafts

                final Map<Path, Map<Path, SQLRowValuesListFetcher>> outerMutable = new HashMap<Path, Map<Path, SQLRowValuesListFetcher>>(f.grafts);

                for (final Entry<Path, Map<Path, SQLRowValuesListFetcher>> e : outerMutable.entrySet()) {

                    final Map<Path, SQLRowValuesListFetcher> innerMutable = new HashMap<Path, SQLRowValuesListFetcher>(e.getValue());

                    for (final Entry<Path, SQLRowValuesListFetcher> innerEntry : innerMutable.entrySet()) {

                        innerEntry.setValue(new SQLRowValuesListFetcher(innerEntry.getValue(), copyTransf));

                    }

                    e.setValue(Collections.unmodifiableMap(innerMutable));

                }

                this.grafts = Collections.unmodifiableMap(outerMutable);

                this.postFetchLinks = f.postFetchLinks;

            }

        }

    }

    /**

     * Get a frozen version of this. If not already {@link #isFrozen() frozen}, copy this and its

     * grafts and {@link #freeze()} the copy.

     *

     * @return <code>this</code> if already frozen, otherwise a frozen copy of <code>this</code>.

     */

    public final SQLRowValuesListFetcher toUnmodifiable() {

        synchronized (this) {

            if (this.isFrozen())

                return this;

            // no need to try to deep copy since we freeze before releasing the lock

            return new SQLRowValuesListFetcher(this, false).freeze();

        }

    }

    /**

     * Make this instance immutable. Ie all setters will now throw {@link IllegalStateException}.

     * Furthermore the request will be computed now once and for all, so as not to be subject to

     * outside modification by {@link #getSelectTransformers()}.

     *

     * @return this.

     */

    public synchronized final SQLRowValuesListFetcher freeze() {

        if (!this.isFrozen()) {

            this.frozen = new SQLSelect(this.getReq());

            for (final Map<Path, SQLRowValuesListFetcher> m : this.grafts.values()) {

                for (final SQLRowValuesListFetcher f : m.values())

                    f.freeze();

            }

        }

        return this;

    }

    public synchronized final boolean isFrozen() {

        return this.frozen != null;

    }

    private final void checkFrozen() {

        if (this.isFrozen())

            throw new IllegalStateException("this has been frozen: " + this);

    }

    /**

     * Whether the rows returned by {@link #fetch()} are {@link SQLRowValues#isFrozen()

     * unmodifiable}.

     *

     * @param b <code>true</code> to make all rows unmodifiable.

     */

    public synchronized final void setReturnedRowsUnmodifiable(final boolean b) {

        this.checkFrozen();

        this.freezeRows = b;

    }

    /**

     * Whether the rows returned by {@link #fetch()} are {@link SQLRowValues#isFrozen()

     * unmodifiable}.

     *

     * @return <code>true</code> if all rows are returned unmodifiable.

     */

    public synchronized boolean areReturnedRowsUnmodifiable() {

        return this.freezeRows;

    }

    public SQLRowValues getGraph() {

        return this.graph;

    }

    public final Path getReferentPath() {

        return this.descendantPath;

    }

    /**

     * Whether to include undefined rows (of tables other than the graph's).

     *

     * @param includeForeignUndef <code>true</code> to include undefined rows.

     */

    public synchronized final void setIncludeForeignUndef(boolean includeForeignUndef) {

        this.checkFrozen();

        this.includeForeignUndef = includeForeignUndef;

    }

    /**

     * Require that only rows with values for the full graph are returned. Eg if the graph is CPI ->

     * OBS, setting this to <code>true</code> will excludes CPI without OBS.

     *

     * @param b <code>true</code> if only full rows should be fetched.

     */

    public synchronized final void setFullOnly(boolean b) {

        this.checkFrozen();

        if (b)

            this.minGraph = this.getGraph().deepCopy();

        else

            this.minGraph = null;

    }

    // MAYBE allow to remove by changing to

    // addRequiredPath(Path)

    // removeRequiredPath(Path)

    // -> just a Set of Path, reduced at the start of fetch()

    public synchronized final void requirePath(final Path p) {

        this.checkFrozen();

        if (this.getGraph().followPath(p) == null)

            throw new IllegalArgumentException("Path not included in this graph : " + p + "\n" + this.getGraph().printGraph());

        if (this.minGraph == null)

            this.minGraph = new SQLRowValues(getGraph().getTable());

        this.minGraph.assurePath(p);

    }

    private synchronized final boolean isPathRequired(final Path p) {

        return this.minGraph != null && this.minGraph.followPath(p) != null;

    }

    private boolean fetchReferents() {

        return this.descendantPath.length() > 0;

    }

    /**

     * To modify the query before execution.

     *

     * @param selTransf will be passed the query which has been constructed, and the return value

     *        will be actually executed, can be <code>null</code>.

     */

    public synchronized void setSelTransf(ITransformer<SQLSelect, SQLSelect> selTransf) {

        this.checkFrozen();

        this.selTransf = selTransf != null ? Collections.singletonList(selTransf) : Collections.<ITransformer<SQLSelect, SQLSelect>> emptyList();

    }

    public void clearSelTransf() {

        this.setSelTransf(null);

    }

    public void appendSelTransf(ITransformer<SQLSelect, SQLSelect> selTransf) {

        this.addSelTransf(selTransf, -1);

    }

    public void prependSelTransf(ITransformer<SQLSelect, SQLSelect> selTransf) {

        this.addSelTransf(selTransf, 0);

    }

    public synchronized void addSelTransf(ITransformer<SQLSelect, SQLSelect> selTransf, final int index) {

        this.checkFrozen();

        if (selTransf != null) {

            final List<ITransformer<SQLSelect, SQLSelect>> copy = new ArrayList<ITransformer<SQLSelect, SQLSelect>>(this.selTransf);

            final int size = copy.size();

            final int realIndex = index < 0 ? size + index + 1 : index;

            copy.add(realIndex, selTransf);

            this.selTransf = Collections.unmodifiableList(copy);

        }

    }

    public synchronized boolean removeSelTransf(ITransformer<SQLSelect, SQLSelect> selTransf) {

        this.checkFrozen();

        if (selTransf != null) {

            final List<ITransformer<SQLSelect, SQLSelect>> copy = new ArrayList<ITransformer<SQLSelect, SQLSelect>>(this.selTransf);

            if (copy.remove(selTransf)) {

                this.selTransf = Collections.unmodifiableList(copy);

                return true;

            }

        }

        return false;

    }

    public synchronized final ITransformer<SQLSelect, SQLSelect> getSelTransf() {

        if (this.selTransf.size() > 1)

            throw new IllegalStateException("More than one transformer");

        return CollectionUtils.getFirst(this.selTransf);

    }

    public synchronized final List<ITransformer<SQLSelect, SQLSelect>> getSelectTransformers() {

        return this.selTransf;

    }

    /**

     * Add a where in {@link #getReq()} to restrict the primary key.

     *

     * @param selID an ID for the primary key, <code>null</code> to not filter.

     */

    public synchronized void setSelID(Number selID) {

        this.checkFrozen();

        this.selID = selID;

    }

    public synchronized final Number getSelID() {

        return this.selID;

    }

    /**

     * Whether to add ORDER BY in {@link #getReq()}.

     *

     * @param b <code>true</code> if the query should be ordered.

     * @return this.

     */

    public synchronized final SQLRowValuesListFetcher setOrdered(final boolean b) {

        this.setOrder(b ? Collections.singleton(Path.get(getGraph().getTable())) : Collections.<Path> emptySet(), true);

        this.setReferentsOrdered(b, false);

        return this;

    }

    public final SQLRowValuesListFetcher setOrder(final List<Path> order) {

        return this.setOrder(order, false);

    }

    private synchronized final SQLRowValuesListFetcher setOrder(final Collection<Path> order, final boolean safeVal) {

        this.checkFrozen();

        for (final Path p : order)

            if (this.getGraph().followPath(p) == null)

                throw new IllegalArgumentException("Path not in this " + p);

        this.ordered = safeVal ? (Set<Path>) order : Collections.unmodifiableSet(new LinkedHashSet<Path>(order));

        return this;

    }

    public synchronized final Set<Path> getOrder() {

        return this.ordered;

    }

    /**

     * Whether to order referent rows in this fetcher.

     *

     * @param b <code>true</code> to order referent rows starting from the primary node, e.g. if the

     *        graph is

     *

     *        <pre>

     * *SITE* <- BATIMENT <- LOCAL

     *        </pre>

     *

     *        then this will cause ORDER BY BATIMENT.ORDRE, LOCAL.ORDRE.

     * @param rec if grafts should also be changed.

     * @return this.

     */

    public synchronized final SQLRowValuesListFetcher setReferentsOrdered(final boolean b, final boolean rec) {

        this.descendantsOrdered = b;

        if (rec) {

            for (final Map<Path, SQLRowValuesListFetcher> m : this.grafts.values()) {

                for (final SQLRowValuesListFetcher f : m.values())

                    f.setReferentsOrdered(b, rec);

            }

        }

        return this;

    }

    public synchronized final boolean areReferentsOrdered() {

        return this.descendantsOrdered;

    }

    public final void addPostFetchLink(final Path toAdd, final Path existingDestination) {

        this.addPostFetchLink(toAdd, existingDestination, false);

    }

    /**

     * Add a link to be added at the end of fetch(). This is needed when the graph to be fetched

     * isn't a tree.

     *

     * @param toAdd the last step of this parameter will be added at the end of {@link #fetch()},

     *        e.g. /SOURCE/ --[ID_MOST_SERIOUS_OBS]--> /OBSERVATION/.

     * @param existingDestination where the destination rows of <code>toAdd</code> are, e.g.

     *        /SOURCE/ <--[ID_SOURCE]-- /SOURCE_OBSERVATION/ --[ID_OBSERVATION]--> /OBSERVATION/.

     * @param ignoreIfMissing what to do if a passed path isn't in this, <code>true</code> to do

     *        nothing, <code>false</code> to throw an exception.

     * @return <code>true</code> if the link was added.

     */

    public synchronized final boolean addPostFetchLink(final Path toAdd, final Path existingDestination, final boolean ignoreIfMissing) {

        checkFrozen();

        if (toAdd.getLast() != existingDestination.getLast())

            throw new IllegalArgumentException("Different destination tables");

        if (!toAdd.isSingleField())

            throw new IllegalArgumentException("Path to add isn't composed of single fields");

        final Step lastStep = toAdd.getStep(-1);

        if (lastStep.getDirection() != Direction.FOREIGN)

            throw new IllegalArgumentException("Last step isn't foreign : " + lastStep);

        if (!getFetchers(toAdd).isEmpty())

            throw new IllegalArgumentException("Path to add already fetched");

        final Path pathToFK = toAdd.minusLast();

        final ListMap<Path, SQLRowValuesListFetcher> fkFetchers = getFetchers(pathToFK);

        if (fkFetchers.isEmpty()) {

            if (ignoreIfMissing)

                return false;

            else

                throw new IllegalArgumentException("Path to add should only have the last step missing");

        }

        final String lastFieldName = lastStep.getSingleField().getName();

        for (final Entry<Path, List<SQLRowValuesListFetcher>> e : fkFetchers.entrySet()) {

            final int pathLength = e.getKey().length();

            for (final SQLRowValuesListFetcher fkFetcher : e.getValue()) {

                if (!fkFetcher.getGraph().followPath(pathToFK.subPath(pathLength)).contains(lastFieldName)) {

                    if (ignoreIfMissing)

                        return false;

                    else

                        throw new IllegalArgumentException("Foreign key " + lastFieldName + " isn't fetched");

                }

            }

        }

        if (getFetchers(existingDestination).isEmpty()) {

            if (ignoreIfMissing)

                return false;

            else

                throw new IllegalArgumentException("Destination won't be fetched : " + existingDestination);

        }

        final Map<Path, Path> copy = new HashMap<>(this.postFetchLinks);

        copy.put(toAdd, existingDestination);

        this.postFetchLinks = Collections.unmodifiableMap(copy);

        return true;

    }

    public synchronized Map<Path, Path> getPostFetchLinks() {

        return this.postFetchLinks;

    }

    public final SQLRowValuesListFetcher graft(final SQLRowValuesListFetcher other) {

        return this.graft(other, Path.get(getGraph().getTable()));

    }

    public final SQLRowValuesListFetcher graft(final SQLRowValues other, Path graftPath) {

        // with referents otherwise it's useless

        return this.graft(new SQLRowValuesListFetcher(other, true), graftPath);

    }

    /**

     * Graft a fetcher on this graph.

     *

     * @param other another instance fetching rows of the table at <code>graftPath</code>.

     * @param graftPath a path from this values to where <code>other</code> rows should be grafted.

     * @return the previous fetcher.

     */

    public synchronized final SQLRowValuesListFetcher graft(final SQLRowValuesListFetcher other, Path graftPath) {

        checkFrozen();

        if (this == other)

            throw new IllegalArgumentException("trying to graft onto itself");

        if (other.getGraph().getTable() != graftPath.getLast())

            throw new IllegalArgumentException("trying to graft " + other.getGraph().getTable() + " at " + graftPath);

        final SQLRowValues graftPlace = this.getGraph().followPath(graftPath);

        if (graftPlace == null)

            throw new IllegalArgumentException("path doesn't exist: " + graftPath);

        assert graftPath.getLast() == graftPlace.getTable();

        if (other.getGraph().hasForeigns())

            throw new IllegalArgumentException("shouldn't have foreign rows");

        final Path descendantPath = other.getReferentPath();

        final int descendantPathLength = descendantPath.length();

        if (descendantPathLength == 0)

            throw new IllegalArgumentException("empty path");

        // checked by computePath

        assert descendantPath.isSingleField();

        // we used to disallow that :

        // this is LOCAL* -> BATIMENT -> SITE and CPI -> LOCAL -> BATIMENT* is being grafted

        // but this is sometimes desirable, e.g. for each LOCAL find all of its siblings with the

        // same capacity (or any other predicate)

        // shallow copy : all values are still immutable

        final Map<Path, Map<Path, SQLRowValuesListFetcher>> outerMutable = new HashMap<Path, Map<Path, SQLRowValuesListFetcher>>(this.grafts);

        final Map<Path, SQLRowValuesListFetcher> innerMutable;

        if (!this.grafts.containsKey(graftPath)) {

            // allow getFetchers() to use a list, easing tests and avoiding using equals()

            innerMutable = new LinkedHashMap<Path, SQLRowValuesListFetcher>(4);

        } else {

            final Map<Path, SQLRowValuesListFetcher> map = this.grafts.get(graftPath);

            innerMutable = new LinkedHashMap<Path, SQLRowValuesListFetcher>(map);

            // e.g. fetching *BATIMENT* <- LOCAL and *BATIMENT* <- LOCAL <- CPI (with different

            // WHERE) and LOCAL have different fields. This isn't supported since we would have to

            // merge fields in merge() and it would be quite long

            for (Entry<Path, SQLRowValuesListFetcher> e : map.entrySet()) {

                final Path fetcherPath = e.getKey();

                final SQLRowValuesListFetcher fetcher = e.getValue();

                for (int i = 1; i <= descendantPathLength; i++) {

                    final Path subPath = descendantPath.subPath(0, i);

                    if (fetcherPath.startsWith(subPath)) {

                        if (!fetcher.getGraph().followPath(subPath).getFields().equals(other.getGraph().followPath(subPath).getFields()))

                            throw new IllegalArgumentException("The same node have different fields in different fetcher\n" + graftPath + "\n" + subPath);

                    } else {

                        break;

                    }

                }

            }

        }

        final SQLRowValuesListFetcher res = innerMutable.put(descendantPath, other);

        outerMutable.put(graftPath, Collections.unmodifiableMap(innerMutable));

        this.grafts = Collections.unmodifiableMap(outerMutable);

        return res;

    }

    public final Collection<SQLRowValuesListFetcher> ungraft() {

        return this.ungraft(Path.get(getGraph().getTable()));

    }

    public synchronized final Collection<SQLRowValuesListFetcher> ungraft(final Path graftPath) {

        checkFrozen();

        if (!this.grafts.containsKey(graftPath))

            return null;

        final Map<Path, Map<Path, SQLRowValuesListFetcher>> outerMutable = new HashMap<Path, Map<Path, SQLRowValuesListFetcher>>(this.grafts);

        final Map<Path, SQLRowValuesListFetcher> res = outerMutable.remove(graftPath);

        this.grafts = Collections.unmodifiableMap(outerMutable);

        return res == null ? null : res.values();

    }

    private synchronized final Map<Path, Map<Path, SQLRowValuesListFetcher>> getGrafts() {

        return this.grafts;

    }

    /**

     * The fetchers grafted at the passed path.

     *

     * @param graftPath where the fetchers are grafted, e.g. MISSION, DOSSIER, SITE.

     * @return the grafts by their path to fetch, e.g. SITE, BATIMENT, LOCAL, CPI_BT.

     */

    public final Map<Path, SQLRowValuesListFetcher> getGrafts(final Path graftPath) {

        return this.getGrafts().get(graftPath);

    }

    /**

     * Get all fetchers.

     *

     * @param includeSelf <code>true</code> to include <code>this</code> (with a <code>null</code>

     *        key).

     * @return all instances indexed by the graft path.

     */

    public final ListMapItf<Path, SQLRowValuesListFetcher> getFetchers(final boolean includeSelf) {

        final ListMap<Path, SQLRowValuesListFetcher> res = new ListMap<Path, SQLRowValuesListFetcher>();

        for (final Entry<Path, Map<Path, SQLRowValuesListFetcher>> e : this.getGrafts().entrySet()) {

            assert e.getKey() != null;

            res.putCollection(e.getKey(), e.getValue().values());

        }

        if (includeSelf)

            res.add(null, this);

        return ListMap.unmodifiableMap(res);

    }

    /**

     * Get instances which fetch the {@link Path#getLast() last table} of the passed path. E.g.

     * useful if you want to add a where to a join. This method is recursively called on

     * {@link #getGrafts(Path) grafts} thus the returned paths may be fetched by grafts.

     *

     * @param fetchedPath a path starting by this table.

     * @return all instances indexed by the graft path, i.e. <code>fetchedPath</code> is between

     *         with it and (it+fetchers.{@link #getReferentPath()}).

     */

    public final ListMap<Path, SQLRowValuesListFetcher> getFetchers(final Path fetchedPath) {

        final ListMap<Path, SQLRowValuesListFetcher> res = new ListMap<Path, SQLRowValuesListFetcher>();

        if (this.getGraph().followPath(fetchedPath) != null)

            res.add(Path.get(getGraph().getTable()), this);

        // search grafts

        for (final Entry<Path, Map<Path, SQLRowValuesListFetcher>> e : this.getGrafts().entrySet()) {

            final Path graftPlace = e.getKey();

            if (fetchedPath.startsWith(graftPlace) && fetchedPath.length() > graftPlace.length()) {

                final Path rest = fetchedPath.subPath(graftPlace.length());

                // we want requests that use the last step of fetchedPath

                assert rest.length() > 0;

                for (final Entry<Path, SQLRowValuesListFetcher> e2 : e.getValue().entrySet()) {

                    final Path refPath = e2.getKey();

                    final SQLRowValuesListFetcher graft = e2.getValue();

                    if (refPath.startsWith(rest)) {

                        res.add(graftPlace, graft);

                    } else if (rest.startsWith(refPath)) {

                        // otherwise rest == refPath and the above if would have been executed

                        assert rest.length() > refPath.length();

                        for (final Entry<Path, List<SQLRowValuesListFetcher>> e3 : graft.getFetchers(rest).entrySet()) {

                            res.addAll(graftPlace.append(e3.getKey()), e3.getValue());

                        }

                    }

                }

            }

        }

        return res;

    }

    private final void addFields(final SQLSelect sel, final SQLRowValues vals, final String alias) {

        // put key first

        final SQLField key = vals.getTable().getKey();

        sel.addSelect(new AliasedField(key, alias));

        for (final String fieldName : vals.getFields()) {

            if (!fieldName.equals(key.getName()))

                sel.addSelect(new AliasedField(vals.getTable().getField(fieldName), alias));

        }

    }

    public final SQLSelect getReq() {

        return this.getReq(null, null);

    }

    static private final SQLSelect checkTr(final List<String> origSelect, final SQLSelect tr) {

        if (!origSelect.equals(tr.getSelect()))

            throw new IllegalArgumentException("Select clause cannot be modified");

        return tr;

    }

    public synchronized final SQLSelect getReq(final Where w, final ITransformer<SQLSelect, SQLSelect> selTransf) {

        checkTable(w);

        final boolean isNopTransf = selTransf == null || selTransf == Transformer.<SQLSelect> nopTransformer();

        if (this.isFrozen()) {

            if (w == null && isNopTransf) {

                return this.frozen;

            } else {

                final SQLSelect copy = new SQLSelect(this.frozen);

                final SQLSelect res = isNopTransf ? copy : checkTr(copy.getSelect(), selTransf.transformChecked(copy));

                return res.andWhere(w);

            }

        }

        final SQLTable t = this.getGraph().getTable();

        final SQLSelect sel = new SQLSelect();

        if (this.includeForeignUndef) {

            sel.setExcludeUndefined(false);

            sel.setExcludeUndefined(true, t);

        }

        walk(null, new ITransformer<State<String>, String>() {

            @Override

            public String transformChecked(State<String> input) {

                final String alias;

                if (input.getFrom() != null) {

                    alias = getAlias(sel, input.getPath());

                    final String aliasPrev = input.getAcc();

                    // MAYBE use "INNER" for first step of a referent graft since the first node is

                    // ignored (the node from the parent graph is used)

                    // SITE <-- BATIMENT and graft is BATIMENT <-- LOCAL, empty BATIMENT are just

                    // discarded so best not to fetch them for nothing

                    final String joinType = isPathRequired(input.getPath()) ? "INNER" : "LEFT";

                    sel.addJoin(joinType, aliasPrev, input.getPath().getStep(-1), alias);

                } else {

                    alias = null;

                }

                addFields(sel, input.getCurrent(), alias);

                return alias;

            }

        });

        for (final Path p : this.getOrder())

            sel.addOrder(sel.followPath(t.getName(), p), false);

        // after getOrder() since it can specify more precise order

        if (this.areReferentsOrdered()) {

            final int descSize = this.descendantPath.length();

            for (int i = 1; i <= descSize; i++) {

                sel.addOrder(sel.followPath(t.getName(), this.descendantPath.subPath(0, i)), false);

            }

        }

        if (this.getSelID() != null)

            sel.andWhere(getIDWhere(this.getSelID()));

        final List<String> origSel = new ArrayList<String>(sel.getSelect());

        SQLSelect res = sel;

        for (final ITransformer<SQLSelect, SQLSelect> tr : this.getSelectTransformers()) {

            res = tr.transformChecked(res);

        }

        if (!isNopTransf)

            res = selTransf.transformChecked(res);

        return checkTr(origSel, res).andWhere(w);

    }

    public final Where getIDWhere(final Number id) {

        if (id == null)

            return null;

        return new Where(getGraph().getTable().getKey(), "=", id);

    }

    static String getAlias(final SQLSelect sel, final Path path) {

        String res = "tAlias";

        final int stop = path.length();

        for (int i = 0; i < stop; i++) {

            res += "__" + path.getSingleField(i).getName();

        }

        // needed for backward, otherwise tableAlias__ID_BATIMENT for LOCAL

        res += "__" + path.getTable(stop).getName();

        return sel.getUniqueAlias(res);

    }

    // assure that the graph is explored the same way for the construction of the request

    // and the reading of the resultSet

    private <S> void walk(final S sel, final ITransformer<State<S>, S> transf) {

        // walk through foreign keys and never walk back (use graft())

        this.getGraph().getGraph().walk(this.getGraph(), sel, transf, RecursionType.BREADTH_FIRST, Direction.FOREIGN);

        // walk starting backwards but allowing forwards

        this.getGraph().getGraph().walk(this.getGraph(), sel, new ITransformer<State<S>, S>() {