SuggestionManager.java

Go to the documentation of this file.

/*
 * Project: Server for annotations sharing
 * Author: Ing. Jaroslav Dytrych idytrych@fit.vutbr.cz
 * File: SuggestionManager.java
 * Description: Class provides offerining of suggestion with usage of local 
 *              knowledge repository.
 */

/**
 * @file SuggestionManager.java
 *
 * @brief Class provides offerining of suggestion from local knowledge 
 *        repository.
 */

package cz.vutbr.fit.knot.annotations.modules.suggestionManager;

import cz.vutbr.fit.knot.annotations.app.AppBean;
import cz.vutbr.fit.knot.annotations.app.Constants;
import cz.vutbr.fit.knot.annotations.comet.EditorSession;
import cz.vutbr.fit.knot.annotations.comet.Linearizer;
import cz.vutbr.fit.knot.annotations.comet.RequestInfo;
import cz.vutbr.fit.knot.annotations.comet.SuggestionLogEntry;
import cz.vutbr.fit.knot.annotations.comet.Util;
import cz.vutbr.fit.knot.annotations.entity.*;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.Comparator;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.Matcher;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.MatcherProvider;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.UpdatableFragment;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.compareMethods.ExactMethod;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.compareMethods.LevenshteinMethod;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.nodeIterators.BidirectionallyUnNestNodeIterator;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.nodeIterators.SequenceNodeIterator;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.nodeIterators.XPathNodeIterator;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.alternative.AlternativeManager;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.attributes.SugAttributeManager;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.attributes.SugBaseAttribute;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.attributes.SugLinkedAttribute;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.attributes.SugNestedAttribute;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.Query;
import javax.xml.xpath.XPathExpressionException;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.w3c.dom.Document;

/**
 * Class provides offerining of suggestion with usage of local knowledge repository.
 *
 * @brief Class provides offerining of suggestions with usage of local knowledge 
 *        repository.
 *
 * @author Martin Petr (xpetrm05)
 */

/* 
 * INFO:
 * If suggestions are created, their types are created in default group 
 * of actual user. For other users suggestions will be handled independently 
 * on types and types will be cloned to their's default group (if they 
 * are already existing, they are used). Confirmation or refusing then will be 
 * in default group so there shhould not be mess in feedback in basic scenario,
 * 
 * TODO: In advanced scenario user will be changing own default group 
 * so feedback have to be used from all user's groups.
 * 
 * TODO: Locking of the document should be revised.
 */

/**
 * Class provides offerining of suggestions.
 * 
 * @author Martin Petr (xpetrm05)
 */
public class SuggestionManager extends Thread {

  /** Informations about request from client*/
  RequestInfo requestInfo = null;
  
  public static final int NOT_USABLE_CONFIDENCE = 4;
  public static final int MINIMAL_CONFIDENCE = 5;
  public static final int INITIAL_CONFIDENCE = 50;
  public static final int REFUSED_CONFIDENCE = 40;
  public static final int CONFIRMED_CONFIDENCE = 90;
  public static final int MINIMAL_FILTER_CONFIDENCE = 5;
  /** Name of this module */
  public static final String MODULE_NAME = "SuggestionManager";
  
  /** Default user group */
  public UserGroup userGroup;
    
  /**
   * Source of suggestions (unknown - probably user, SEC API or NER).
   * 
   * @brief Source of suggestions.
   */
  public enum SuggestionSource {
    NONE,
    SEC_API,
    NER;
  }
  
  /**
   * Constructor
   * 
   * @param request Informations about client request
   */
  public SuggestionManager(RequestInfo request){
    super("SuggestionsManagerModuleThread");
    this.requestInfo = request;
  }

  /**
   * Method start the thread code execution. 
   */
  @Override
  public void run() { 
    if (requestInfo.getSession() == null || requestInfo.getSession().getUser() == null 
        || requestInfo.getSession().getUser().getGroups() == null || requestInfo.getSession().getUser().getGroups().isEmpty()
        || requestInfo.getSession().getSyncDocument() == null || requestInfo.getSession().getDefaultGroup() == null) {
      return;
    }
    
    AnnotDocument doc = requestInfo.getSession().getSyncDocument();
    
    if(doc != null){
      while(true){
        if(AppBean.getLockMaster().getDocumentLock(doc.getId()).lockForRead()){
          break;
        }
      }
    }
    
    try{
      startModule();
    }
    
    catch(Exception ex){
      String msg = "Unknown exception in SuggestionManager: " + ex.getMessage();
      Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg, ex);
      // unlock document for change
      if(doc != null){
        AppBean.getLockMaster().getDocumentLock(doc.getId()).unlockForRead();
      }
    } 
    
    finally{
      // unlock document for change
      if(doc != null){
        AppBean.getLockMaster().getDocumentLock(doc.getId()).unlockForRead();
      }
    }
  }  // run()
  
  /**
   * Method start module
   */
  void startModule(){
    
    if (isDocUpdate(requestInfo)) {
      while (true) {
        if (AppBean.getLockMaster().getSuggReqLock(requestInfo.getSession().getSyncDocument().getId()).lockForChange()) {
          break;
        }
      }
      try {
        updateDocument(requestInfo);
      } finally {
        AppBean.getLockMaster().getSuggReqLock(requestInfo.getSession().getSyncDocument().getId()).unlockForChange();
      }
    }
    
    if (isSuggestionsRequest(requestInfo) && !isPositiveFeedback(requestInfo)) {
      // request for new suggestions
      String result = getSuggestions(requestInfo);
      requestInfo.getSession().addMessageTS(result);
      requestInfo.getSession().notifyComet();
    }
    
    // positive feedback, added annotations, changed annotations
    ArrayList<SuggestionLogEntry> rSugs = new ArrayList<SuggestionLogEntry>();
    ArrayList<SuggestionLogEntry> sSugs = new ArrayList<SuggestionLogEntry>();
    if (isPositiveFeedback(requestInfo)) {
      rSugs = refuseOtherSugFromFragments(requestInfo);
    }
    if (!requestInfo.getSession().getUpdateNegFeedbackForRefusedBySame()) {
      sSugs = rSugs;
    }
    
    if (isNegativeFeedback(requestInfo)) {
      // suggestions were refused
      rejectSuggestions(requestInfo, sSugs, rSugs);
    }
    
    if (requestInfo.getAlternativesRequestedFor() != null && !requestInfo.getAlternativesRequestedFor().isEmpty()) {
      boolean logIt = AppBean.getLogGetAlternativesFor();
      // alternatives for some suggestions was requested
      Iterator<Suggestion> sugIt = requestInfo.getAlternativesRequestedFor().iterator();
      while (sugIt.hasNext()) {
        Suggestion sug = sugIt.next();
        AlternativeManager.activateAlternatives(requestInfo, sug);
        if (logIt) {
          String msg = "getAlternativesFor " + sug.getId() + " for document " + requestInfo.getSession().getSyncDocument().getUri();
          Logger.getLogger(SuggestionManager.class.getName()).log(Level.WARNING, msg);
        }
      }
      String result = printSuggestionsDiff(requestInfo.getSession(), new ArrayList<Suggestion>(), new ArrayList<AnnotType>());
      requestInfo.getSession().addMessageTS(result);
      requestInfo.getSession().notifyComet();
    }
    
    if (isAnnotDelete(requestInfo)) {
      // annotations was deleted
      String result = annotationsDelete(requestInfo);
      requestInfo.getSession().addMessageTS(result);
      requestInfo.getSession().notifyComet();
    }
    
    // positive feedback, added annotations, changed annotations
    if (isPositiveFeedback(requestInfo) 
         || isAnnotChange(requestInfo) 
         || isAnnotAdded(requestInfo)) {
      // make message for client
      annotationsAddChange(requestInfo);
    }
  }
  
  /**
   * Top level method thats handle situation when there is positive feedback, 
   * added annotations or changed annotations
   * 
   * @param request informations about client request
   */
  private void annotationsAddChange(RequestInfo request) {
    // get list of added and edited annotations
    ArrayList<Annotation> addedAnnotations = new ArrayList<Annotation>();
    ArrayList<Annotation> editedAnnotations = new ArrayList<Annotation>();
    addedAnnotations.addAll(requestInfo.getFlier().getAddedAnnotations());
    editedAnnotations.addAll(requestInfo.getFlier().getEditedAnnotations());

    // lock document for suggestion change 
    lockDocForChange(requestInfo.getSession().getSyncDocument().getId());
    ArrayList<Suggestion> suggestionsToChange = SugPersister.persist(requestInfo, addedAnnotations, editedAnnotations, null, true, new ArrayList<AnnotType>());
    AlternativeManager.confirmSuggestions(request, suggestionsToChange);
    // unlock document for suggestions change
    unlockDocForChange(requestInfo.getSession().getSyncDocument().getId());
    
    
    String result = printSuggestionsDiff(requestInfo.getSession(), suggestionsToChange, new ArrayList<AnnotType>());
    requestInfo.getSession().addMessageTS(result);
    requestInfo.clearConfirmedSuggestions();
    requestInfo.getSession().notifyComet();  // send to client
    
    synchronized(AppBean.getSessions()){
      Iterator<EditorSession> sIt = AppBean.getSessions().iterator();
      while (sIt.hasNext()) {  // send to others
        EditorSession es = sIt.next();
        if (es.equals(requestInfo.getSession()) || es.getSyncDocument() == null || !es.getSyncDocument().equals(request.getSession().getSyncDocument())) {
          continue;
        }
        result = printSuggestionsDiff(es, suggestionsToChange, new ArrayList<AnnotType>());
        es.addMessageTS(result);
        es.notifyComet();
      }
    }
  }  // annotationsAddChange()
  
  /**
   * Method finds all linked and nested suggestions of suggestions and return
   * them as an ArrayList.
   * 
   * @param suggestion suggestion with possible dependencies
   * @return list of found linked and nested suggestions
   */
  private static ArrayList<Suggestion> getAllDependencies(Suggestion suggestion){
    ArrayList<Suggestion> retArray = new ArrayList<Suggestion>(); 
    if(suggestion != null){
      
      // go trough all attributes of suggestion
      List attrObjects = suggestion.getAttributes();
      if(attrObjects != null && !attrObjects.isEmpty()){
        Iterator attrIt = attrObjects.iterator();
        while(attrIt.hasNext()){
          Object actualObject = (Object) attrIt.next();
          
          // try find linked suggestion attributes
          if(actualObject instanceof SugLinkedAttribute){
            // if find linked suggestion attribute
            SugLinkedAttribute currentLink = (SugLinkedAttribute)actualObject;
            if(currentLink.getValue() != null && currentLink.getValue() instanceof Suggestion){
              // add it to return array
              retArray.add(((Suggestion)currentLink.getValue()));
            }
          }
          // find all nested suggestions
          else if (actualObject instanceof SugNestedAttribute) {
            SugNestedAttribute currentNested = (SugNestedAttribute)actualObject;
            if (currentNested.getValue() != null && currentNested.getValue() instanceof Suggestion) {
              retArray.add(((Suggestion)currentNested.getValue()));
            }
          }
        }
      }
      // if the suggestion is nested in another one, the parent suggestion
      // must be added to the list as a dependency
      if (suggestion.getNestedInSuggestion() != null) {
        retArray.add(suggestion.getNestedInSuggestion());
      }
    }  // if(suggestion != null){
    
    return retArray;
  }  // getAllDependencies()
  

  /**
   * Top level method thats handle situation when annotations are deleted.
   * 
   * @param request informations about client request
   * @return string with message for client
   */
  private String annotationsDelete(RequestInfo request){
    // get annotations deleted by user
    ArrayList<Suggestion> changedSuggestions = getAffectedForDelete(request.getFlier().getRemovedAnnotations());

    return printSuggestionsDiff(request.getSession(), changedSuggestions, new ArrayList<AnnotType>()); 
  }
  
  /**
   * Top level method thats handle situation when user want suggestions.
   * 
   * @param request informations about client request
   * @return string with message for client
   */
  private String getSuggestions(RequestInfo request){
    while(true){
      if(AppBean.getLockMaster().getSuggReqLock(requestInfo.getSession().getSyncDocument().getId()).lockForChange()){
        break;
      }
    }
    try {
      ArrayList<Suggestion> allSuggest = getSuggestionsFromDB(request.getSession().getSyncDocument().getId());
      if (allSuggest.isEmpty()) {
        updateDocument(request);
        return "";
      }
    } finally {
      AppBean.getLockMaster().getSuggReqLock(requestInfo.getSession().getSyncDocument().getId()).unlockForChange();
    }
    return printSuggestionsDiff(request.getSession(), new ArrayList<Suggestion>(), new ArrayList<AnnotType>());
  }  // getSuggestions()
  
  /**
   * Top level method thats handle situation when synchronized document was updated.
   * 
   * @param request informations about client request
   */
  private void updateDocument(RequestInfo request){
    // suggestions from DB
    ArrayList<Suggestion> oldSuggestions = new ArrayList<Suggestion>();
    ArrayList<Suggestion> addedSuggestions = new ArrayList<Suggestion>();
    ArrayList<Suggestion> changedSuggestions = new ArrayList<Suggestion>();
    ArrayList<AnnotType> newTypes = new ArrayList<AnnotType>();
    
    // get suggestions from DB
    oldSuggestions.addAll(getSuggestionsFromDB(request.getSession().getSyncDocument().getId()));
    // update document and suggestions belongs to document with new suggestions from NER
    updateDocument(request,oldSuggestions,addedSuggestions,changedSuggestions, newTypes);
  
    ArrayList<Suggestion> modifiedSuggestions = new ArrayList<Suggestion>();
    modifiedSuggestions.addAll(addedSuggestions);
    modifiedSuggestions.addAll(changedSuggestions);
    
    String result = printSuggestionsDiff(request.getSession(), modifiedSuggestions, newTypes);
    request.getSession().addMessageTS(result);
    request.getSession().notifyComet();
    
    synchronized(AppBean.getSessions()){
      Iterator<EditorSession> sIt = AppBean.getSessions().iterator();
      while (sIt.hasNext()) {  // send to others
        EditorSession es = sIt.next();
        if (es.equals(request.getSession()) || es.getSyncDocument() == null || !es.getSyncDocument().equals(request.getSession().getSyncDocument())) {
          continue;
        }
        result = printSuggestionsDiff(es, modifiedSuggestions, new ArrayList<AnnotType>());  // types was already sent
        es.addMessageTS(result);
        es.notifyComet();
      }
    }
  }  // updateDocument()
  
  /**
   * Send message to other clients
   * 
   * @param requestInfo Informations about client request
   * @param message String with message
   */
  public static void sendToOthers(RequestInfo requestInfo, String message) {
    synchronized(AppBean.getSessions()){
      Iterator<EditorSession> sesIt = AppBean.getSessions().iterator();
      long sessionId = requestInfo.getSession().getSessionId();
      while (sesIt.hasNext()) {
        EditorSession es = sesIt.next();
        if (es.getSessionId() == sessionId) {
          continue;
        }
        es.addMessageTS(message);
      }
    }
  }
  
  /**
   * Top level method thats handle situation when user rejected some suggestions.
   * 
   * @param request Informations about client request
   * @param skipSugs Suggestions for which update of negative feedback should be skipped
   * @param noAlternatives List of suggestions for which alternatives should not be activated
   */
  private void rejectSuggestions(RequestInfo request, ArrayList<SuggestionLogEntry> skipSugs, ArrayList<SuggestionLogEntry> noAlternatives){
    // get suggestions directly affected with rejection
    ArrayList<Suggestion> suggestionsToDelete = updateNegativeFeedback(request, skipSugs, noAlternatives);
    
    ArrayList<Suggestion> allSug = request.getSession().getWholeSendedSuggestions();
    suggestionsToDelete.addAll(getNestedSuggFromToDelete(allSug, suggestionsToDelete));
    ArrayList<Suggestion> affectedSug = cutRefusedSuggLinks(allSug, suggestionsToDelete);
    affectedSug.addAll(suggestionsToDelete);
    
    String result = printSuggestionsDiff(request.getSession(), affectedSug, new ArrayList<AnnotType>());
    requestInfo.getSession().addMessageTS(result);
    requestInfo.getSession().notifyComet();
    synchronized(AppBean.getSessions()){
      Iterator<EditorSession> sIt = AppBean.getSessions().iterator();
      while (sIt.hasNext()) {  // send to others
        EditorSession es = sIt.next();
        if (es.equals(requestInfo.getSession()) || es.getSyncDocument() == null || !es.getSyncDocument().equals(request.getSession().getSyncDocument())) {
          continue;
        }
        result = printSuggestionsDiff(es, affectedSug, new ArrayList<AnnotType>());
        es.addMessageTS(result);
        es.notifyComet();
      }
    }
  }  // rejectSuggestions()
  
  /**
   * Creates message with added annotation types for the client
   * 
   * @param session Informations about client session
   * @param newTypes Array with new types of annotations
   * @param requestInfo Informations about client request
   * @return Returns message for the client
   */
  public static String printTypes(RequestInfo requestInfo, EditorSession session, ArrayList<AnnotType> newTypes) {
    boolean withAnc = true;  // types will be send with ancestors
    if (requestInfo.getSession().getProtocolLOD() < Constants.PROTOCOL_LOD_V1_1) {
      withAnc = false;  // old client - types will be send without ancestors
    }
    String addedTStr = "";
    Iterator<AnnotType> typeIter = newTypes.iterator();
    while (typeIter.hasNext()) {  // for each added type of annotation
      AnnotType type = typeIter.next();
      if (isTypeInGroups(type, requestInfo)) {
        // TODO: If client not interested, don't send type (use session.getQueriedTypes())
        if (requestInfo.getSession().getProtocolLOD() < Constants.PROTOCOL_LOD_V2) {
          addedTStr = addedTStr + type.toXMLString(withAnc);
        } else {
          addedTStr = addedTStr + type.toXMLStringV2(withAnc);
        }
      }
    }
    if (!addedTStr.isEmpty()) {  // if there are any added types to send
      if (requestInfo.getSession().getProtocolLOD() < Constants.PROTOCOL_LOD_V2) {
        addedTStr = "<types><add>" + addedTStr + "</add></types>";  // create message
      } else {
        addedTStr = "<addTypes>" + addedTStr + "</addTypes>";  // create message
      }
    }
    if (!addedTStr.isEmpty()) {  // if there is something to send
      return addedTStr;
    } else {
      return "";
    }
  }
  
  /**
   * Checks whether type come under user groups of logged in user
   *
   * @param annotType Checked type of annotation
   * @param requestInfo Informations about client request
   * @return If type come under user groups of logged in user, returns true,
   * false otherwise
   */
  public static boolean isTypeInGroups(AnnotType annotType, RequestInfo requestInfo) {
    if (requestInfo.getSession().getUser() == null) {  // if user is not logged in, there is no groups
      return false;
    }
    if (annotType.getGroup() == null) { // if type of annotation isn't assigned to user group
      return false;  // it can't come under user groups
    }
    Iterator<UserGroup> grIter = requestInfo.getSession().getUser().getGroups().iterator();
    while (grIter.hasNext()) {  // for each user group of logged in user
      UserGroup userGroup = grIter.next();
      if (annotType.getGroup().equals(userGroup)) {  // if group of type is matching this group
        return true;
      }
    }
    return false;
  }  // isTypeInGroups()

  
  /**
   * Method which prepares messages for the client and updates client's session
   * based on now modified suggestions and previous client's session.
   * 
   * @param session Informations about client session
   * @param modifiedSuggestions Suggestions which was modified in this request
   * @param newTypes List of new annotation types to send along with suggestions
   * @return Returns string with message for client
   */
  public String printSuggestionsDiff(EditorSession session, ArrayList<Suggestion> modifiedSuggestions, ArrayList<AnnotType> newTypes) {
    ArrayList<Suggestion> shouldHave = null;
    synchronized (session) {
      ArrayList<Suggestion> allSuggest = getSuggestionsFromDB(session.getSyncDocument().getId());
      ArrayList<Suggestion> filteredSuggest = new ArrayList<Suggestion>(allSuggest);
      filtrateSuggestions(requestInfo, session, filteredSuggest);
      shouldHave = addAllDependencies(filteredSuggest, allSuggest);
      ArrayList<Suggestion> removedSug = getSuggestionWithoutFeedback(shouldHave, modifiedSuggestions);
      ArrayList<Suggestion> rSug = getNestedSuggFromToDelete(shouldHave, removedSug);
      modifiedSuggestions.addAll(cutRefusedSuggLinks(shouldHave, rSug));
      modifiedSuggestions.addAll(removedSug);
      checkTypes(session, shouldHave, newTypes, true);
    }
    persistTypes(newTypes);
    // message with types
    String msgTypes = printTypes(requestInfo, session, newTypes);
    // send types to others
    sendToOthers(requestInfo, msgTypes);
    return msgTypes + filterByClientCache(session, shouldHave, modifiedSuggestions);
  }
  
  /**
   * Gets true value if one suggestion have link to other.
   * 
   * @param suggestion suggestion where will be link searched
   * @param linkedSuggestion suggestion to which is link searched 
   * @return Returns true value if one suggestion have link to other, false otherwise.
   */
  private boolean haveLinkOn(Suggestion suggestion, Suggestion linkedSuggestion){
    if(suggestion != null && linkedSuggestion != null){
      // get attributes of suggestion
      List attrList = suggestion.getAttributes();
      if(attrList != null && !attrList.isEmpty()){
        
        // go trough all objects that represents attributes
        Iterator attrIt = attrList.iterator();
        while(attrIt.hasNext()){
          
          Object actualAttr = (Object) attrIt.next();
          // test if it is linked attribute
          if(actualAttr instanceof SugLinkedAttribute){
            
            // test if attribute points to Suggestion
            SugLinkedAttribute actualLink= (SugLinkedAttribute) actualAttr;
            if(actualLink.getValue() instanceof Suggestion){
              
              // test if suggestion is searched suggestion
              if(((Suggestion)actualLink.getValue()).getId().equals(linkedSuggestion.getId())){
                return true;
              }
            }
          } // if(actualLink.getValue() instanceof Suggestion)
        } // while(attrIt.hasNext())
      } // if(attrList != null && !attrList.isEmpty())
    }
    
    return false;
  }  // haveLinkOn()
  
  /**
   * Initializes Requested types definitions in AppBean
   */
  public static void initReqTypes() {
    ArrayList<SecApiReqTypeDef> availableTypes = null;
    try {
      SECAPIInterface sai = AppBean.getSecApiInterface();
      String SECAPIResults = sai.makeRequest(createSecApiTypesRequest());
      // process types and attributes
      availableTypes = SECTypesTranslator.translate(SECAPIResults);
    } catch (Exception ex) {
      String message = "SEC API call error : " + ex.getMessage();
      Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, message);
      AppBean.setReqTypeDefinitions(new ArrayList<SecApiReqTypeDef>());  // clear list
      return;
    }
    if (availableTypes == null || availableTypes.isEmpty()) {
      AppBean.setReqTypeDefinitions(new ArrayList<SecApiReqTypeDef>());  // clear list
      return;
    }
    
    AppBean.setAvTypeDefinitions(availableTypes);
    // load desired and unnecessary type definitions from database
    loadTypeDefsFromDB();
    // filter types and prepare AppBean structures
    SecApiReqTypeDef.filterTypes();
  }  // initReqTypes()
  
  /**
   * Initializes Requested types definitions in AppBean from remote SEC API
   * 
   * @param conn Connection to SEC API server
   * @param requestInfo Info about clients request
   */
  public static void initReqTypesFromRemote(SECAPIConn conn, RequestInfo requestInfo) {
    ArrayList<SecApiReqTypeDef> availableTypes = null;
    try {
      String SECAPIResults = conn.sendReqToSECAPI(createSecApiTypesRequest(), requestInfo);
      // process types and attributes
      availableTypes = SECTypesTranslator.translate(SECAPIResults);
    } catch (Exception ex) {
      String message = "SEC API call error : " + ex.getMessage();
      Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, message);
      AppBean.setReqTypeDefinitions(new ArrayList<SecApiReqTypeDef>());  // clear list
      return;
    }
    if (availableTypes == null || availableTypes.isEmpty()) {
      AppBean.setReqTypeDefinitions(new ArrayList<SecApiReqTypeDef>());  // clear list
      return;
    }
    
    AppBean.setAvTypeDefinitions(availableTypes);
    // load desired and unnecessary type definitions from database
    loadTypeDefsFromDB();
    // filter types and prepare AppBean structures
    SecApiReqTypeDef.filterTypes();
  }  // initReqTypesFromRemote()
  
  /**
   * Loads desired and unnecessary type definitions from database
   */
  @SuppressWarnings("unchecked")
  private static void loadTypeDefsFromDB() {
    String desiredJSONData = "";
    String unnecessaryJSONData = "";
    String unnecessaryAtJSONData = "";
    
    String desiredEAJSONData = "";
    String unnecessaryEAJSONData = "";
    String unnecessaryAtEAJSONData = "";
    
    Object[] paramsD = {"name", Constants.SEC_DESIRED_TYPES_SETTINGS_NAME};
    List<ServerSetting> settingsList = AppBean.getPersistenceManager().queryDB("ServerSetting.findByName", paramsD);
    if (settingsList != null && !settingsList.isEmpty()) {
      if (!settingsList.get(0).getSettingValue().isEmpty()) {
        desiredJSONData = settingsList.get(0).getSettingValue();
      }
    }
    Object[] paramsU = {"name", Constants.SEC_UNNECESSARY_TYPES_SETTINGS_NAME};
    settingsList = AppBean.getPersistenceManager().queryDB("ServerSetting.findByName", paramsU);
    if (settingsList != null && !settingsList.isEmpty()) {
      if (!settingsList.get(0).getSettingValue().isEmpty()) {
        unnecessaryJSONData = settingsList.get(0).getSettingValue();
      }
    }
    Object[] paramsUA = {"name", Constants.SEC_UNNECESSARY_TYPES_SETTINGS_NAME};
    settingsList = AppBean.getPersistenceManager().queryDB("ServerSetting.findByName", paramsUA);
    if (settingsList != null && !settingsList.isEmpty()) {
      if (!settingsList.get(0).getSettingValue().isEmpty()) {
        unnecessaryAtJSONData = settingsList.get(0).getSettingValue();
      }
    }
    
    Object[] paramsDEA = {"name", Constants.SEC_DESIRED_TYPES_SETTINGS_NAME_EA};
    settingsList = AppBean.getPersistenceManager().queryDB("ServerSetting.findByName", paramsDEA);
    if (settingsList != null && !settingsList.isEmpty()) {
      if (!settingsList.get(0).getSettingValue().isEmpty()) {
        desiredEAJSONData = settingsList.get(0).getSettingValue();
      }
    }
    Object[] paramsUEA = {"name", Constants.SEC_UNNECESSARY_TYPES_SETTINGS_NAME_EA};
    settingsList = AppBean.getPersistenceManager().queryDB("ServerSetting.findByName", paramsUEA);
    if (settingsList != null && !settingsList.isEmpty()) {
      if (!settingsList.get(0).getSettingValue().isEmpty()) {
        unnecessaryEAJSONData = settingsList.get(0).getSettingValue();
      }
    }
    Object[] paramsUAEA = {"name", Constants.SEC_UNNECESSARY_TYPES_SETTINGS_NAME};
    settingsList = AppBean.getPersistenceManager().queryDB("ServerSetting.findByName", paramsUAEA);
    if (settingsList != null && !settingsList.isEmpty()) {
      if (!settingsList.get(0).getSettingValue().isEmpty()) {
        unnecessaryAtEAJSONData = settingsList.get(0).getSettingValue();
      }
    }
    
    ArrayList<SecApiReqTypeDef> desiredTypes = SECTypesTranslator.translate(desiredJSONData);
    ArrayList<SecApiReqTypeDef> unnecessaryTypes = SECTypesTranslator.translate(unnecessaryJSONData);
    ArrayList<SecApiReqTypeDef> unnecessaryAttributes = SECTypesTranslator.translate(unnecessaryAtJSONData);
    
    ArrayList<SecApiReqTypeDef> desiredTypesEA = SECTypesTranslator.translate(desiredEAJSONData);
    ArrayList<SecApiReqTypeDef> unnecessaryTypesEA = SECTypesTranslator.translate(unnecessaryEAJSONData);
    ArrayList<SecApiReqTypeDef> unnecessaryAttributesEA = SECTypesTranslator.translate(unnecessaryAtEAJSONData);
    
    AppBean.setDesTypeDefinitions(desiredTypes);
    AppBean.setUnTypeDefinitions(unnecessaryTypes);
    AppBean.setUnAtDefinitions(unnecessaryAttributes);
    
    AppBean.setDesTypeDefinitionsEA(desiredTypesEA);
    AppBean.setUnTypeDefinitionsEA(unnecessaryTypesEA);
    AppBean.setUnAtDefinitionsEA(unnecessaryAttributesEA);
  }  // loadTypeDefsFromDB()
  
  /**
   * Gets suggestions from SEC API and translates them to the Suugestion objects.
   * 
   * @param parRequestInfo Informations about client request
   * @param doc Linearized document
   * @param uri URI of original document
   * @param useAlternatives True for work with alternatives, false otherwise
   * @param validSuggestions List of valid suggestions
   * @return Returns suggestions (ArrayList<Suggestion>) or alternatives of suggestions (ArrayList<Alternative>) for the document 
   */
  public String getSuggestionsFromSECAPI(RequestInfo parRequestInfo, String doc, String uri, boolean useAlternatives, ArrayList<Suggestion> validSuggestions){
    String secApiXmlData = null;
    
    String SECAPIResults = "";
    
    String secApiRequest = createSecApiSuggestionsRequest(doc, uri, useAlternatives);
    
    Boolean processed = false;
    try {
      SECAPIInterface sai = AppBean.getSecApiInterface();
      SECAPIResults = sai.makeRequest(secApiRequest);
      processed = true;
    } catch (Exception ex) {
      String message = "SEC API call error : " + ex.getMessage();
      Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, message);
    }

    // Second attempt
    if (SECAPIResults == null || SECAPIResults.isEmpty()) {
      try {
        SECAPIInterface sai = AppBean.getSecApiInterface();
        // restart SEC API before process
        sai.close();
        if (!sai.init()) {
          Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, "Initialisation of SEC API failed!");
          return secApiXmlData;
        }
        // it is possible that we have updated version of SEC API now
        // (old version was killed and new was started now)
        initReqTypes();
        if (AppBean.getReqTypeDefinitions().isEmpty()) {
          return secApiXmlData;
        }
        // update request data
        secApiRequest = createSecApiSuggestionsRequest(doc, uri, false);
        // process request
        SECAPIResults = sai.makeRequest(secApiRequest);
        processed = true;
      } catch (Exception ex) {
        String message = "SEC API call error : " + ex.getMessage();
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, message);
        return secApiXmlData;
      }
    }

    // get XML from JSON
    secApiXmlData = getSuggestionsXMLFromJSON(SECAPIResults);
    if (secApiXmlData.isEmpty()) {
      return null;
    }
    return secApiXmlData;
  }  // getSuggestionsFromSECAPI()
  
  /**
   * Gets suggestions from remote SEC API and translates them to the Suggestion objects.
   * 
   * @param parRequestInfo Informations about client request
   * @param doc Linearized document
   * @param uri URI of original document
   * @param useAlternatives True for work with alternatives, false otherwise
   * @param validSuggestions List of valid suggestions
   * @return Returns suggestions (ArrayList<Suggestion>) or alternatives of suggestions (ArrayList<Alternative>) for the document 
   */
  public String getSuggestionsFromRemoteSECAPI(RequestInfo parRequestInfo, String doc, String uri, boolean useAlternatives, ArrayList<Suggestion> validSuggestions){
    String secApiXmlData = null;
    
    String SECAPIResults = "";
    
    String secApiRequest = "";
    
    try {
      SECAPIConn sac = new SECAPIConn();
      // it is possible that SEC API was updated after last request
      // (old version was killed and new was started) so we need to refresh types
      initReqTypesFromRemote(sac, parRequestInfo);
      if (AppBean.getReqTypeDefinitions().isEmpty()) {
        return secApiXmlData;
      }
      // create request data
      secApiRequest = createSecApiSuggestionsRequest(doc, uri, useAlternatives);
      // process request


      SECAPIResults = sac.sendReqToSECAPI(secApiRequest, parRequestInfo);
    } catch (Exception ex) {
      String message = "Remote SEC API call error : " + ex.getMessage();
      Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, message);
    }
    
    if(SECAPIResults == null){
        return null;
    }
    
    // get XML from JSON
    secApiXmlData = getSuggestionsXMLFromJSON(SECAPIResults);
    if (secApiXmlData.isEmpty()) {
      return null;
    }
    return secApiXmlData;
  }  // getSuggestionsFromRemoteSECAPI()
  
  /**
   * Gets XML with suggestions from JSON response data from SEC API
   * 
   * @param responseData Response data from SEC API
   * @return Returns XML with suggestions in String
   */
  private static String getSuggestionsXMLFromJSON(String responseData) {
    JSONObject jsonData = (JSONObject)JSONValue.parse(responseData);
    if (jsonData == null) {
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = "Unable to parse received JSON string with suggestions.";
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
      }
      return "";
    }
    if (checkException(jsonData)) {  // check for exception
      return "";
    }
    String suggXML = null;
    // try to get XML with suggestions
    try {
      suggXML = (String) jsonData.get("annotation");
    } catch (Exception e) {
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = "Unable to parse received JSON string with suggestions.";
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg, e);
      }
      return "";
    }
    
    return suggXML;
  }  // getSuggestionsXMLFromJSON()
  
  /**
   * Checks if JSON from SEC API contains exception and if yes, logs it
   * 
   * @param jsonData JSON data from SEC API
   * @return If exception was presented, returns true, false otherwise
   */
  private static boolean checkException(JSONObject jsonData) {
    String exceptionData = null;
    try {  // try to get exception
      exceptionData = (String) jsonData.get("exception");
    } catch (Exception e) {  // error never mind here
    }
    if (exceptionData != null && !exceptionData.isEmpty()) {
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = "SEC API returned exception: " + exceptionData;
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
      }
      return true;
    }
    return false;
  }
  
  /**
   * Creates SEC API request for suggestions
   *    
   * @param doc Linearized document
   * @param uri URI of original document
   * @param useAlternatives True for work with alternatives, false otherwise
   * @return Returns SEC API request for suggestions
   */
  private static String createSecApiSuggestionsRequest(String doc, String uri, boolean useAlternatives) {
    StringBuilder reqTypes = new StringBuilder();
    ArrayList<SecApiReqTypeDef> rtdList = AppBean.getReqTypeDefinitions();
    if (rtdList == null) {
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = "Unable to get requseted type definitions while creating "
                   + "SEC API Suggestions Request.";
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
      }
      rtdList = new ArrayList<SecApiReqTypeDef>();
    }
    Iterator<SecApiReqTypeDef> rtdIt= rtdList.iterator();
    if (rtdIt.hasNext()) {  // first type
      SecApiReqTypeDef rtd = rtdIt.next();
      reqTypes.append(rtd.getRequestString());
    }
    while (rtdIt.hasNext()) {
      SecApiReqTypeDef rtd = rtdIt.next();
      reqTypes.append(", ");
      reqTypes.append(rtd.getRequestString());
    }
    
    return "{ \"annotate\": {" +
      " \"input_text\": \"" + EscapeForSecApi(doc) + "\"," +
      " \"annotation_format\": [\"sxml\"]," +
      " \"disambiguate\": " + (useAlternatives ? "0" : "1") + "," +
      " \"document_uri\": \"" + uri + "\"," +
      " \"types_and_attributes\": { " + reqTypes + " }"+ 
      " } }\n";
  }
  private static String disambiguate = "1";
  
  /**
   * Creates SEC API request for types and attributes
   * 
   * @return Returns SEC API request for types and attributes
   */
  private static String createSecApiTypesRequest() {
    return "{ \"get_entity_types_and_attributes\": {} }\n";
  }
  
  /**
   * Finds offsets of all ocurrences of refuse border separator in linearized 
   * document (separators are defined in Constants)
   * 
   * @param doc Linearized document
   * @return Returns offsets of separators
   */
  private static ArrayList<Integer> findSeparators(String doc) {
    ArrayList<Integer> result = new ArrayList<Integer>();
    Iterator<String> sepIt = Constants.REFUSE_BORDER_SEPARATOR.iterator();
    while (sepIt.hasNext()) {
      String sep = sepIt.next();
      int offset = doc.indexOf(sep);
      while (offset >= 0) {
        result.add(offset);
        offset = doc.indexOf(sep, offset + 1);
      }
    }
    Collections.sort(result);
    return result;
  }  // findSeparators()
  
  
  /**
   * Method for updating negative feedback
   * 
   * @param request Informations about client request
   * @param skipSugs Suggestions for which update should be skipped
   * @param noAlternatives List of suggestions for which alternatives should not be activated
   * @return Refused suggestions
   */
  private ArrayList<Suggestion> updateNegativeFeedback(RequestInfo request, ArrayList<SuggestionLogEntry> skipSugs, ArrayList<SuggestionLogEntry> noAlternatives){
    ArrayList<SuggestionLogEntry> suggLog = request.getRefusedSuggestions();
    ArrayList<Suggestion> changedSuggestions = new ArrayList<Suggestion>();
    Integer groupId = request.getSession().getDefaultGroup().getId();

    ArrayList<SuggestionLogEntry> addToLog = new ArrayList<SuggestionLogEntry>();
    
    ArrayList<Suggestion> allSugg = request.getSession().getWholeSendedSuggestions();
    if (allSugg != null && !allSugg.isEmpty()) {  // if there are some sended suggestions
      HashMap<Integer, ArrayList<SuggestionFragment>> sugToLinFragments = new HashMap<Integer, ArrayList<SuggestionFragment>>();
      SECAPISuggestionTranslator.linearizeSuggestions(sugToLinFragments, request.getSession().getParsedSyncDocument(), allSugg);
      ArrayList<Integer> separators = findSeparators(request.getSession().getParsedLinDocument());
      int docLength = request.getSession().getParsedLinDocument().length();
      
      Iterator<SuggestionLogEntry> refSugIt = suggLog.iterator();
      while (refSugIt.hasNext()) {  // for each refused suggestion
        SuggestionLogEntry refSugLogEntry = refSugIt.next();
        if (skipSugs.contains(refSugLogEntry)) {
          continue;
        }

        // get refused suggestion's annotated text
        Integer refusedSuggestionId = Integer.parseInt(refSugLogEntry.getTmpId());
        Suggestion refusedSuggestion = getSuggestionFromDB(refusedSuggestionId);
        ArrayList<SuggestionFragment> suggFragList = refusedSuggestion.getFragmentsAL();
        AnnotType refusedSuggType = refusedSuggestion.getAnnotType();

        // get refusedURI
        String refusedURI = refusedSuggestion.getSAEntityIdentifier();
        if (refusedURI != null && !refusedURI.isEmpty()) {  // if there is URI of Entity
          if (suggFragList != null && !suggFragList.isEmpty()) {  // if suggestion have at least one fragment
            if (suggFragList.size() >= 2) {
              Collections.sort(suggFragList, new SuggestionFragmentComp());
            }
            
            // find separators for current suggestion
            int leftSep = 0;
            int rightSep = docLength;
            Integer curOffset = null;
            ArrayList<SuggestionFragment> frags = sugToLinFragments.get(refusedSuggestionId);
            SuggestionFragment frag = null;
            if (frags != null && !frags.isEmpty()) {
              frag = frags.get(0);
            } else {
              frag = null;  // no fragments - maybe it is for whole document
            }
            if (frag != null) {
              curOffset = frag.getOffset();
              Iterator<Integer> sepIt = separators.iterator();
              while (sepIt.hasNext()) {
                Integer s = sepIt.next();
                if (s < curOffset && leftSep < s) {
                  leftSep = s;
                }
                if (s > curOffset && s < rightSep) {
                  rightSep = s;
                }
              }
            }
            
            Iterator<Suggestion> allSuggIt = allSugg.iterator();
            while (allSuggIt.hasNext()) {  // for each sended suggestion
              Suggestion actualSugg = allSuggIt.next();
              
              // Find offset for comparison with separators
              frags = sugToLinFragments.get(actualSugg.getId());
              if (frags != null && !frags.isEmpty()) {
                frag = frags.get(0);
              } else {
                frag = null;
              }
              if (frag != null) {
                curOffset = frag.getOffset();
              } else {
                curOffset = null;
              }
              
              // condition to ensure that sugg. are the same type and sugg. with the same id will be skipped 
              Integer actualSuggId = actualSugg.getId();
              AnnotType actualSuggType = actualSugg.getAnnotType();
              String actualURI = actualSugg.getSAEntityIdentifier();
              if (actualSuggId != null && refusedSuggestionId != null && !actualSuggId.equals(refusedSuggestionId)
                      && actualSuggType != null && refusedSuggType != null && actualSuggType.equals(refusedSuggType)
                      && actualURI != null && refusedURI.equals(actualURI)) {
                // if it is not same suggestion but have same type and SEC API Entity URI
                ArrayList<SuggestionFragment> tmpFragList = actualSugg.getFragmentsAL();
                if (tmpFragList.size() >= 2) {
                  Collections.sort(tmpFragList, new SuggestionFragmentComp());
                }
                // check whether both suggestion fragment lists have the same amount of fragments
                if (tmpFragList.size() == suggFragList.size()) {
                  Iterator<SuggestionFragment> tmpFragListIt = tmpFragList.iterator();
                  Iterator<SuggestionFragment> suggFragListIt = suggFragList.iterator();
                  Boolean errFlag = false;
                  // condition to check if !tmpFragList.isEmpty() is not needed
                  while (tmpFragListIt.hasNext()) {
                    // same annotated text
                    SuggestionFragment tmpFrag1 = tmpFragListIt.next();
                    SuggestionFragment tmpFrag2 = suggFragListIt.next();
                    if (!(tmpFrag1.getAnnotatedText()).equals(tmpFrag2.getAnnotatedText())) {
                      errFlag = true;
                      break;
                    }
                    // different path and offset
                    String path1 = tmpFrag1.getPath();
                    String path2 = tmpFrag2.getPath();
                    if (path1 != null && path2 != null && path1.equals(path2)) {
                      Integer offset1 = tmpFrag1.getOffset();
                      Integer offset2 = tmpFrag2.getOffset();
                      if (offset1 != null && offset2 != null && offset1.equals(offset2)) {
                        errFlag = true;
                        break;
                      }
                    }
                  }
                  if (curOffset != null) {
                    if (curOffset < leftSep || curOffset > rightSep) {
                      errFlag = true;  // out of borders given by separators
                    }
                  }
                  if (!errFlag) {
                    SuggestionLogEntry tmpLogEntry = new SuggestionLogEntry((actualSugg.getId()).toString());
                    tmpLogEntry.setRefuseMethod(Constants.REFUSED_AUTOMATICALLY);
                    addToLog.add(tmpLogEntry);
                  }
                }
              } // if it is not same suggestion but have same type and SEC API Entity URI
            }  // for each sended suggestion
          }  // if suggestion have at least one fragment
        }  // if there is URI of Entity
      }  // for each refused suggestion
    }  // if there are some sended suggestions
    
    suggLog.addAll(addToLog);  // add additional suggestions found
    
    if(suggLog != null && !suggLog.isEmpty()){
      Iterator<SuggestionLogEntry> logIt = suggLog.iterator();
      while(logIt.hasNext()){
        SuggestionLogEntry actualLogItem = logIt.next();
        lockDocForChange(request.getSession().getSyncDocument().getId());
        boolean createAlternatives = true;
        if (noAlternatives.contains(actualLogItem)) {
          createAlternatives = false;
        }
        updateSuggNegativeFeedback(request, groupId, Integer.parseInt(actualLogItem.getTmpId()), createAlternatives);
        unlockDocForChange(request.getSession().getSyncDocument().getId());
      }
    }
    
    return changedSuggestions;
  }  // updateNegativeFeedback()
  
  /**
   * Refuses other suggestions for fragments for which is some suggestion confirmed
   * 
   * @param request Informations about client request
   * @return Returns list of refused suggestions
   */
  private ArrayList<SuggestionLogEntry> refuseOtherSugFromFragments(RequestInfo request) {
    ArrayList<SuggestionLogEntry> addToLog = new ArrayList<SuggestionLogEntry>();
    ArrayList<SuggestionLogEntry> confSugs = request.getConfirmedSuggestions();
    
    ArrayList<Suggestion> allSugg = request.getSession().getWholeSendedSuggestions();
    if (allSugg != null && !allSugg.isEmpty()) {  // if there are some sended suggestions
      HashMap<Integer, ArrayList<SuggestionFragment>> sugToLinFragments = new HashMap<Integer, ArrayList<SuggestionFragment>>();
      SECAPISuggestionTranslator.linearizeSuggestions(sugToLinFragments, request.getSession().getParsedSyncDocument(), allSugg);
      ArrayList<Integer> separators = findSeparators(request.getSession().getParsedLinDocument());
      int docLength = request.getSession().getParsedLinDocument().length();
      
      Iterator<SuggestionLogEntry> confSugIt = confSugs.iterator();
      while (confSugIt.hasNext()) {  // for each confirmed suggestion
        SuggestionLogEntry confSugLogEntry = confSugIt.next();

        // get confirmed suggestion's annotated text
        Integer confirmedSuggestionId = Integer.parseInt(confSugLogEntry.getTmpId());
        Suggestion confirmedSuggestion = getSuggestionFromDB(confirmedSuggestionId);
        ArrayList<SuggestionFragment> suggFragList = confirmedSuggestion.getFragmentsAL();

        if (suggFragList != null && !suggFragList.isEmpty()) {  // if suggestion have at least one fragment
          if (suggFragList.size() >= 2) {
            Collections.sort(suggFragList, new SuggestionFragmentComp());
          }

          Iterator<Suggestion> allSuggIt = allSugg.iterator();
          while (allSuggIt.hasNext()) {  // for each sended suggestion
            Suggestion actualSugg = allSuggIt.next();

            // sugg. with the same id will be skipped 
            Integer actualSuggId = actualSugg.getId();
            if (actualSuggId != null && confirmedSuggestionId != null && !actualSuggId.equals(confirmedSuggestionId)) {
              // if it is not same suggestion
              ArrayList<SuggestionFragment> tmpFragList = actualSugg.getFragmentsAL();
              if (tmpFragList.size() >= 2) {
                Collections.sort(tmpFragList, new SuggestionFragmentComp());
              }
              // check whether both suggestion fragment lists have the same amount of fragments
              if (tmpFragList.size() == suggFragList.size()) {
                Iterator<SuggestionFragment> tmpFragListIt = tmpFragList.iterator();
                Iterator<SuggestionFragment> suggFragListIt = suggFragList.iterator();
                Boolean errFlag = false;
                // condition to check if !tmpFragList.isEmpty() is not needed
                while (tmpFragListIt.hasNext()) {
                  // same annotated text
                  SuggestionFragment tmpFrag1 = tmpFragListIt.next();
                  SuggestionFragment tmpFrag2 = suggFragListIt.next();
                  if (!(tmpFrag1.getAnnotatedText()).equals(tmpFrag2.getAnnotatedText())) {
                    errFlag = true;
                    break;
                  }
                  // same path
                  String path1 = tmpFrag1.getPath();
                  String path2 = tmpFrag2.getPath();
                  if (path1 != null && path2 != null && !path1.equals(path2)) {
                    errFlag = true;
                    break;
                  }
                  // same offset
                  Integer offset1 = tmpFrag1.getOffset();
                  Integer offset2 = tmpFrag2.getOffset();
                  if (offset1 != null && offset2 != null && !offset1.equals(offset2)) {
                    errFlag = true;
                    break;
                  }
                }
                if (!errFlag) {
                  SuggestionLogEntry tmpLogEntry = new SuggestionLogEntry((actualSugg.getId()).toString());
                  tmpLogEntry.setRefuseMethod(Constants.REFUSED_AUTOMATICALLY);
                  addToLog.add(tmpLogEntry);
                }
              }
            } // if it is not same suggestion
          }  // for each sended suggestion
        }  // if suggestion have at least one fragment
      }  // for each confirmed suggestion
    }  // if there are some sended suggestions
    
    request.getRefusedSuggestions().addAll(addToLog);
    return addToLog;
  }  //refuseOtherSugFromFragments()
  
  
  /**
   * Class which implements Comparator to sort arrayList of SuggestionFragment alphabetically
   * 
   * @brief Class which implements Comparator to sort arrayList of SuggestionFragment alphabetically
   */
  class SuggestionFragmentComp implements java.util.Comparator<SuggestionFragment>{

    /**
     * Compares two fragments according to offset and length
     * 
     * @param frag1 First fragment
     * @param frag2 Second fragment
     * @return Returns a negative integer, zero, or a positive integer as first 
     *         fragment is less than, equal to, or greater than the second fragment. 
     */
    @Override
    public int compare(SuggestionFragment frag1, SuggestionFragment frag2){
      return (frag1.getAnnotatedText()).compareTo(frag2.getAnnotatedText());
    }
  }

  /**
   * Method for updating negative feedback for one suggestion
   * 
   * @param request Informations about client request
   * @param groupId Group ID of the current user
   * @param refusedSuggID ID of the refused Suggestion
   * @param processAlternatives Flag to processing alternatives
   * @return Reference to the updated suggestion
   */
  public static Suggestion updateSuggNegativeFeedback(RequestInfo request, Integer groupId, int refusedSuggID, boolean processAlternatives){
    SuggestionFeedback feedback = getFeedbackSFromDB(refusedSuggID, groupId);
    if(feedback != null){
      feedback.setAnnot(null);
      feedback.setAnnotationId(null);
      feedback.getSuggestion().setConfidence(REFUSED_CONFIDENCE);
      Suggestion sug4update = feedback.getSuggestion();
      Iterator<SugBaseAttribute> attrIt = sug4update.getAttributes().iterator();
      while(attrIt.hasNext()){
        SugBaseAttribute attr = attrIt.next();
        if(attr.getNestedSuggestion() != null){
          Suggestion updatedSugg = updateSuggNegativeFeedback(request, groupId, attr.getNestedSuggestion().getId(), processAlternatives);
          attr.setNestedSuggestion(updatedSugg);
        }
      }
      AppBean.getPersistenceManager().saveEntityChanges(sug4update);
      updateFeedback(feedback);
      return sug4update;
    }else{
      Suggestion actualSuggestion = getSuggestionFromDB(refusedSuggID);
      if(actualSuggestion != null){
        feedback = new SuggestionFeedback();
        feedback.setAnnot(null);
        feedback.setAnnotationId(null);
        feedback.setGroup(request.getSession().getDefaultGroup());
        feedback.setGroupId(request.getSession().getDefaultGroup().getId());
        feedback.setSuggestion(actualSuggestion);
        feedback.setSuggestionId(actualSuggestion.getId());
        actualSuggestion.setConfidence(REFUSED_CONFIDENCE);
        Iterator<SugBaseAttribute> attrIt = actualSuggestion.getAttributes().iterator();
        while(attrIt.hasNext()){
          SugBaseAttribute attr = attrIt.next();
          if(attr.getNestedSuggestion() != null){
            Suggestion updatedSugg = updateSuggNegativeFeedback(request, groupId, attr.getNestedSuggestion().getId(), processAlternatives);
            attr.setNestedSuggestion(updatedSugg);
          }
        }
        AppBean.getPersistenceManager().saveEntityChanges(actualSuggestion);
        persistFeedback(feedback);
        if (processAlternatives) {
          AlternativeManager.activateAlternatives(request, actualSuggestion);
        }
        return actualSuggestion;
      }
      else{
        return null;
      }
    }
  }
  
  /**
   * Main method to update suggestion database after document change. Method 
   * retrieves the new suggestions and compares them with existing old suggestions 
   * and save changes and new suggestions.
   * 
   * @param request informations about client request
   * @param oldSuggestions existing suggestions for document that are in DB
   * @param addedSuggestions there will be new annotations saved
   * @param changedSuggestions there will be changed annotations saved
   * @param newTypes Array for adding new types of annotations
   */
  private void updateDocument(RequestInfo request, ArrayList<Suggestion> oldSuggestions, ArrayList<Suggestion> addedSuggestions, ArrayList<Suggestion> changedSuggestions, ArrayList<AnnotType> newTypes){
    userGroup = requestInfo.getSession().getDefaultGroup();
    if (userGroup == null) {
      return;
    }
    String rawNewSuggestions = null;
    ArrayList<Suggestion> newSuggestions = null;
    String document = request.getSession().getParsedLinDocument();
    String uri = request.getSession().getSyncDocument().getUri();
    
    // select source of suggestions
    SuggestionSource suggestionSource = sourceOfSuggestions(document);
    
    // get suggestions as raw data
    if (suggestionSource == SuggestionSource.SEC_API) {
      if (Constants.USE_REMOTE_SEC_API || SECAPIInterface.isSecCommandEmpty()) {
        rawNewSuggestions = getSuggestionsFromRemoteSECAPI(request, document, uri, false, null);
      } else {
        rawNewSuggestions = getSuggestionsFromSECAPI(request, document, uri, false, null);
      }
    }
    if (rawNewSuggestions == null) {
      return;
    }
    
    // lock document for suggestion change 
    lockDocForChange(requestInfo.getSession().getSyncDocument().getId());
    
    // process suggestions
    try{ // thread have to unlock doc for change
      // convert raw data to suggestions
      if (suggestionSource == SuggestionSource.SEC_API) {
        SECAPISuggestionTranslator st = new SECAPISuggestionTranslator(userGroup);
        newSuggestions = st.translateSugFromSecApi(request, rawNewSuggestions, newTypes);
      }

      if(newSuggestions != null && !newSuggestions.isEmpty()){
        // update fragments of suggestions
        updateSuggestionsFragments(oldSuggestions,request, changedSuggestions);
        // some suggestions may be devastated be updateSuggestionsFragments,
        // fragments of more suggestions may be same, only one of this suggestions should have high confidence
        compareFragments(oldSuggestions, changedSuggestions);
        // save updated suggestions
        SugPersister.persist(request, null, null, changedSuggestions, false, newTypes);

        // get suggestions from DB
        oldSuggestions.clear();
        oldSuggestions.addAll(getSuggestionsFromDB(request.getSession().getSyncDocument().getId()));

        // check if suggestions have right types, if don't have, convert it and hold it
        checkTypes(request.getSession(), oldSuggestions, newTypes, false);
        checkTypes(request.getSession(), changedSuggestions, newTypes, true);
        persistTypes(newTypes);

        // insert new suggestions from NER to DB
        insertNewSuggestions(newSuggestions,oldSuggestions,addedSuggestions,changedSuggestions);

        // persis new suggestions 
        SugPersister.persist(request, null, null, addedSuggestions, false, newTypes);
        // save updated suggestions
        SugPersister.persist(request, null, null, changedSuggestions, false, newTypes);

        // process alternatives of suggestions
        ArrayList<Suggestion> validSuggestions = getSuggestionsForAlternatives(request.getSession().getSyncDocument().getId(), MINIMAL_CONFIDENCE);
        AlternativeManager alternativesThread = new AlternativeManager(request, document, uri, validSuggestions, this, requestInfo.getSession().getSyncDocument().getId());
        alternativesThread.start();
      }
    } catch(Exception ex){
      String msg = "Unknown exception in SuggestionManager: " + ex.getMessage();
      Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg, ex);
    } 
    // unlock document for suggestions change
    unlockDocForChange(requestInfo.getSession().getSyncDocument().getId());
  }  // updateDocument()

  /**
   * Return source of suggestions.
   * 
   * @param document Document.
   * @return Source of suggestions.
   */
  public static SuggestionSource sourceOfSuggestions(String document) {
    SuggestionSource suggestionSource = SuggestionSource.NONE; 
    if (document != null && !document.isEmpty()) {
      if (Constants.SEC_API_SUGGESTIONS) {
        suggestionSource = SuggestionSource.SEC_API;
      }
    }
    return suggestionSource;
  }
  /**
   * Method make message with suggestions from given array.
   * 
   * @param suggestions list of suggestions
   * @return string with added suggestions
   */
  private String getAddMessagesString(ArrayList<Suggestion> suggestions){
    Collections.sort(suggestions, new ConfidenceComparator());
    StringBuilder result = new StringBuilder();
    if(suggestions != null && !suggestions.isEmpty()){
      ArrayList<String> attFilter = requestInfo.getSession().getAttFilter();
      Boolean KBRefMode = requestInfo.getSession().getKBRefMode();
      int lNum = requestInfo.getSession().getLanguageNum();
      Iterator<Suggestion> suggIt = suggestions.iterator();
      while(suggIt.hasNext()){
        Suggestion actualSugg = suggIt.next();
        if(this.requestInfo.getSession().getProtocolLOD() == Constants.PROTOCOL_LOD_V2){
          result.append(actualSugg.toXMLStringV2(attFilter, lNum, KBRefMode));
        }else{
          Integer id = actualSugg.getId();
          String params = " tmpId=\"" + ((id == null) ? "null" : id.toString()) + "\" confidence=\"" + actualSugg.getConfidence().toString() + "\"";
          result.append(actualSugg.toXMLString(true, params, false));
        }
      }
    }
    
    return result.toString();
  }
  
  /**
   * @brief The object implements comparator of Alternative objects that are
   * compared by confidence (from highest to lowest confidence).
   */
  public class ConfidenceComparator implements java.util.Comparator<Suggestion> {

    @Override
    public int compare(Suggestion s1, Suggestion s2) {
      int result = s1.getConfidence().compareTo(s2.getConfidence());
      if (result == 0) {
        return result;
      } else {
        return (result * (-1));
      }
    }
  } // public class ConfidenceComparator
  
  /**
   * Method make message with suggestions to delete from given array.
   * 
   * @param suggestions list of suggestions to delete
   * @return string with suggestions to delete
   */
  private String getDeleteMessagesString(ArrayList<Integer> suggestions){
    StringBuilder result = new StringBuilder();
    if(suggestions != null && !suggestions.isEmpty() && requestInfo.getSession().getProtocolLOD() < Constants.PROTOCOL_LOD_V2){
      Iterator<Integer> suggIt = suggestions.iterator();
      while(suggIt.hasNext()){
        result.append("<delete tmpId=\"");
        result.append(suggIt.next().toString());
        result.append("\"/>");
      }
    } else if (suggestions != null && !suggestions.isEmpty()) {
      ArrayList<Integer> skipSug = new ArrayList<Integer>();
      if (requestInfo.getOrigRefusedSuggestions() != null && !requestInfo.getOrigRefusedSuggestions().isEmpty()) {
        Iterator<SuggestionLogEntry> sleIt = requestInfo.getOrigRefusedSuggestions().iterator();
        while (sleIt.hasNext()) {
          SuggestionLogEntry sle = sleIt.next();
          Integer sid;
          try {
            sid = Integer.parseInt(sle.getTmpId());
          } catch (NumberFormatException numberFormatException) {
            continue;
          }
          skipSug.add(sid);
        }
      }
      if (requestInfo.getConfirmedSuggestions() != null && !requestInfo.getConfirmedSuggestions().isEmpty()) {
        Iterator<SuggestionLogEntry> sleIt = requestInfo.getConfirmedSuggestions().iterator();
        while (sleIt.hasNext()) {
          SuggestionLogEntry sle = sleIt.next();
          Integer sid;
          try {
            sid = Integer.parseInt(sle.getTmpId());
          } catch (NumberFormatException numberFormatException) {
            continue;
          }
          skipSug.add(sid);
        }
      }
      Iterator<Integer> suggIt = suggestions.iterator();
      while(suggIt.hasNext()){
        Integer sugId = suggIt.next();
        if (skipSug.contains(sugId)) {
          continue;
        }
        result.append("<suggestion uri=\"");
        result.append(AppBean.getBaseUri());
        result.append("/sugg/");
        result.append(sugId.toString());
        result.append("\"/>");
      }
    }
    
    return result.toString();
  }

  /**
   * Method make from added suggestions string and added suggestion strings message 
   * for client.
   * 
   * @param suggestionToAddM string with added suggestions
   * @param suggestionToRemoveM string with removed suggestions
   */
  private String makeClientMessage(String suggestionToAddM,String suggestionToRemoveM){
    StringBuilder result = new StringBuilder();
    if(!suggestionToAddM.isEmpty() || !suggestionToRemoveM.isEmpty()){
      
      if(requestInfo.getSession().getProtocolLOD() == Constants.PROTOCOL_LOD_V2){
        result.append("<removeSuggestions>");
        result.append(suggestionToRemoveM);
        result.append("</removeSuggestions>");
        result.append("<addSuggestions>");
        result.append(suggestionToAddM);
        result.append("</addSuggestions>");
      }else{
        result.append("<suggestions>");
        result.append(suggestionToRemoveM);
        result.append(suggestionToAddM);
        result.append("</suggestions>");
      }
    } else if (requestInfo.getSession().getProtocolLOD() == Constants.PROTOCOL_LOD_V2 
               && requestInfo.getSession().getSugRequestFlag()) {
      result.append("<addSuggestions/>");
    }
    return result.toString();
  }
  
  private void getNewClientCache(ArrayList<Suggestion> addedSugg, ArrayList<Integer> deletedSugg, ArrayList<Integer> actualCache){
    // starts with deleted
    if(deletedSugg != null && !deletedSugg.isEmpty()){
      Iterator<Integer> deletedIt = deletedSugg.iterator();
      while(deletedIt.hasNext()){
        Integer actualForDelete = deletedIt.next();
        if(actualCache.contains(actualForDelete)){
          actualCache.remove(actualForDelete);
        }
      }
    }
    
    // check adds
    if(addedSugg != null && !addedSugg.isEmpty()){
      Iterator<Suggestion> addedIt = addedSugg.iterator();
      while(addedIt.hasNext()){
        Suggestion actualForAdd = addedIt.next();
        if(!actualCache.contains(actualForAdd.getId())){
          actualCache.add(actualForAdd.getId());
        }
      }
    }
  }
  
  /**
   * Method filter list with suggestions which client should have and leave only 
   * suggestions that are interested for client (new or changed). Client's session 
   * is updated and message for client generated.
   * 
   * @param session Informations about client session
   * @param shouldHave List with suggestions which client should have
   * @param changedSuggestions List with changed suggestions
   * @return String with messages for client
   */
  private String filterByClientCache(EditorSession session, ArrayList<Suggestion> shouldHave, ArrayList<Suggestion> changedSuggestions){
    ArrayList<Suggestion> sugToAdd = new ArrayList<Suggestion>();
    ArrayList<Integer> sugToDelete = new ArrayList<Integer>();
    
    synchronized (session) {
      // create clientCach list
      ArrayList<Integer> clientCacheID = session.getSendedSuggestions();
      ArrayList<Suggestion> clientCache = new ArrayList<Suggestion>();
      for (Iterator<Integer> it = clientCacheID.iterator(); it.hasNext();) {
        Integer id = it.next();
        clientCache.add(getSuggestionFromDB(id));
      }

      ArrayList<Suggestion> helpList = new ArrayList<Suggestion>();
      helpList.addAll(clientCache);
      for (Iterator<Suggestion> it = helpList.iterator(); it.hasNext();) {
        Suggestion sg = it.next();
        if ((!shouldHave.contains(sg)) || changedSuggestions.contains(sg)) {
          if (!sugToDelete.contains(sg.getId())) {
            sugToDelete.add(sg.getId());
          }
          clientCache.remove(sg);
        }
      }

      for (Iterator<Suggestion> it = shouldHave.iterator(); it.hasNext();) {
        Suggestion sg = it.next();
        if (!clientCache.contains(sg)) {
          clientCache.add(sg);
          if (!sugToAdd.contains(sg)) {
            sugToAdd.add(sg);
          }
        }
      }

      ArrayList<Suggestion> closed = new ArrayList<Suggestion>(sugToAdd);
      ArrayList<Suggestion> visited = new ArrayList<Suggestion>();
      while (!closed.isEmpty()) {
        Suggestion actualSugg = closed.get(0);
        visited.add(actualSugg);
        closed.remove(0);
        Iterator<Suggestion> linkedSuggIt = getAllDependencies(actualSugg).iterator();
        while (linkedSuggIt.hasNext()) {
          Suggestion actualLinkedSugg = linkedSuggIt.next();
          if (clientCache.contains(actualLinkedSugg)) {
            if (!sugToDelete.contains(actualLinkedSugg.getId())) {
              sugToDelete.add(actualLinkedSugg.getId());
            }
            if (!sugToAdd.contains(actualLinkedSugg)) {
              sugToAdd.add(actualLinkedSugg);
            }
            if (!visited.contains(actualLinkedSugg)) {
              closed.add(actualLinkedSugg);
            }
          }
        }
      }

      getNewClientCache(sugToAdd, sugToDelete, clientCacheID);
      session.setSendedSugg(clientCacheID);

      for (Iterator<Suggestion> sugIt = sugToAdd.iterator(); sugIt.hasNext();) {
        Suggestion suggestion = sugIt.next();
        if (suggestion.getNestedInAnnot() != null || suggestion.getNestedInSuggestion() != null) {
          sugIt.remove();
        }
      }

      String addMessage = getAddMessagesString(sugToAdd);
      String removMessage = getDeleteMessagesString(sugToDelete);
      String clientMessage = makeClientMessage(addMessage, removMessage);
      return clientMessage;

    }  // synchronized
  }  // filterByClientCache()

  /**
   * Method filter document suggestions by user group feedback, method builds 
   * suggestion list without suggestions with negative references.
   * 
   * @param request request from user
   * @param docSuggestions all document suggestions
   */
  private ArrayList<Suggestion> filtrateRefusedSugg(RequestInfo request, ArrayList<Suggestion> docSuggestions){
    Integer documentId = request.getSession().getSyncDocument().getId();
    Integer userGroupId = request.getSession().getDefaultGroup().getId();
    ArrayList<Suggestion> result = new ArrayList<Suggestion>();
    
    // Get negative feedback for current document and group
    ArrayList<SuggestionFeedback> negFeedback = getNegFeedbackFromDB(documentId,userGroupId);
    if(negFeedback != null && !negFeedback.isEmpty() && docSuggestions != null && !docSuggestions.isEmpty()){
      
      // Check all suggestions if is refused by group
      Iterator<Suggestion> docSugIt = docSuggestions.iterator();
      while(docSugIt.hasNext()){
        Suggestion actualSuggestion = docSugIt.next();
        Iterator<SuggestionFeedback> negFeedbackIt = negFeedback.iterator();
        while(negFeedbackIt.hasNext()){
          // If suggestion is in negative feedback list
          if(actualSuggestion.getId().equals(negFeedbackIt.next().getSuggestionId())){
            // Add suggestion to result list
            result.add(actualSuggestion);
            // Delete it from suggestions for user
            docSugIt.remove();
          }
        }
      }
    }
    
    return result;
  }  // filtrateRefusedSugg()
  
  /**
   * Return suggestion from list by id. Return null if not found.
   * 
   * @param suggestionId Id of suggestion
   * @param list List of suggestion
   * @return Founded suggestion
   */
  public static Suggestion getSuggestionFromList(Integer suggestionId, List<Suggestion> list) {
    if (suggestionId == null || list == null) {
      return null;
    }
    for (Iterator<Suggestion> it = list.iterator(); it.hasNext();) {
      Suggestion sg = it.next();
      if (sg.getId() == null) {
        continue;
      }
      if (sg.getId().equals(suggestionId)) {
        return sg;
      }
    }
    return null;
  }
  
  /**
   * Removes suggestions with feedback from the list and replaces all references 
   * to them by annotations from feedback (for negative feedback by null).
   * 
   * @param list List of suggestions
   * @param modified List to which modified suggestions will be added
   * @return Removed suggestions from list.
   */
  private ArrayList<Suggestion> getSuggestionWithoutFeedback(ArrayList<Suggestion> list, ArrayList<Suggestion> modified) {
    ArrayList<Suggestion> removedSuggestions = new ArrayList<Suggestion>();
    UserGroup defGroup = requestInfo.getSession().getDefaultGroup();
    Integer defaultUserGroupID = null;
    if (defGroup != null) {
      defaultUserGroupID = defGroup.getId();
    } else {
      if (this.userGroup != null) {
        defaultUserGroupID = this.userGroup.getId();
      } else {
        // if it is not possible to get feedback, it is better to remove all
        removedSuggestions.addAll(list);
        list.removeAll(list);
        return removedSuggestions;
      }
    }
    
    List<SuggestionFeedback> feedList = getFeedbackFromDB(requestInfo.getSession().getSyncDocument().getId(), defaultUserGroupID);
    ArrayList<SuggestionFeedback> feedListOtherGroups = new ArrayList<SuggestionFeedback>();
    
    if (requestInfo.getSession().getUser() != null) {
      Iterator<UserGroup> allUserGroups = requestInfo.getSession().getUser().getGroupsAL().iterator();
      while(allUserGroups.hasNext()){
        UserGroup group = allUserGroups.next();
        if(group.getId() != defaultUserGroupID){
           List<SuggestionFeedback> feed = getFeedbackFromDB(requestInfo.getSession().getSyncDocument().getId(), group.getId());
           feedListOtherGroups.addAll(feed);
        }
      }
    }
    
    // delete suggestions with positiveFeedback in default group
    for (Iterator<SuggestionFeedback> it = feedList.iterator(); it.hasNext();) {
      SuggestionFeedback fb = it.next();
      if (fb.getSuggestion() == null) {
        continue;
      }
      Suggestion sgFromList = getSuggestionFromList(fb.getSuggestion().getId(), list);
      if (sgFromList != null) { // suggestion was found
        feedbackChangeRef(list, sgFromList, modified, fb, removedSuggestions);
      }
    }
    
    // delete suggestions with positiveFeedback in other group
    for (Iterator<SuggestionFeedback> it = feedListOtherGroups.iterator(); it.hasNext();) {
      SuggestionFeedback fb = it.next();
      if (fb.getSuggestion() == null) {
        continue;
      }
      Suggestion sgFromList = getSuggestionFromList(fb.getSuggestion().getId(), list);
      if (sgFromList != null) { // suggestion was found
        feedbackChangeRef(list, sgFromList, modified, fb, removedSuggestions);
      }
    }
    
    return removedSuggestions;
  }  // getSuggestionWithoutFeedback()
  
  // iterate all other suggestion and change references
  
  /**
   * Adds links in suggestion to modified suggs
   * 
   * @param list List of suggestions
   * @param sgFromList Suggestion to check for links
   * @param modified List to which modified suggestions will be added
   * @param fb feedback for this suggestion
   * @param removedSuggestions Removed suggestions from list.
   */
  private void feedbackChangeRef(ArrayList<Suggestion> list, 
          Suggestion sgFromList, ArrayList<Suggestion> modified, SuggestionFeedback fb,
          ArrayList<Suggestion> removedSuggestions)
  {
    for (Iterator<Suggestion> itList = list.iterator(); itList.hasNext(); ) {
      Suggestion sg = itList.next();
      List atrs = sg.getAttributes();
      // iterate in all atributes
      for (Iterator it2 = atrs.iterator(); it2.hasNext();) {
        Object obj = (Object) it2.next();
        Object value = null;
        if (obj instanceof SugLinkedAttribute) {
          value = ((SugLinkedAttribute) obj).getValue();
        }
        if (obj instanceof SugNestedAttribute) {
          value = ((SugNestedAttribute) obj).getValue();
        }
        if (value instanceof Suggestion) {
          if (((Suggestion) value).equals(sgFromList)) {
            if (obj instanceof SugLinkedAttribute) {
              ((SugLinkedAttribute) obj).setValue(fb.getAnnot());
              if (modified != null && !modified.contains(sg)) {
                modified.add(sg);
              }
            }
            if (obj instanceof SugNestedAttribute) {
              ((SugNestedAttribute) obj).setValue(fb.getAnnot());
              if (modified != null && !modified.contains(sg)) {
                modified.add(sg);
              }
            }
          }
        }
      }
    }
    list.remove(sgFromList);
    removedSuggestions.add(sgFromList);
  }
  
  /**
   * Finds suggestion in feedback
   * 
   * @param feedbackList List with feedback
   * @param searchedSugg Searched suggestion
   * @return If suggestion is in feedback, returns true, false otherwise.
   */
  private boolean isSuggestionInFeedback(ArrayList<SuggestionFeedback> feedbackList, Suggestion searchedSugg){
    if(feedbackList != null){
      Iterator<SuggestionFeedback> feedbackgIterator = feedbackList.iterator();
      while(feedbackgIterator.hasNext()){
        SuggestionFeedback actualFeedback = feedbackgIterator.next();
        if(actualFeedback.getAnnot() != null && actualFeedback.getSuggestionId().equals(searchedSugg.getId())){
          return true;
        }
      }
    }
    
    return false;
  }
  
  /** 
   * Method searches for those suggestions from the input array that are nested
   * in ones included in the second array.
   * 
   * @param inputSuggs All current suggestions
   * @param suggsToDelete Suggestions that are going removed
   * @return Suggestions that are nested in removed suggestions
   */
  public static ArrayList<Suggestion> getNestedSuggFromToDelete(ArrayList<Suggestion> inputSuggs, ArrayList<Suggestion> suggsToDelete) {
    ArrayList<Suggestion> toRet = new ArrayList<Suggestion>();
    if (!inputSuggs.isEmpty()) {
      Iterator<Suggestion> inputSuggsIt = inputSuggs.iterator();
      // for every suggestion
      while (inputSuggsIt.hasNext()) {
        Suggestion inputSug = inputSuggsIt.next();
        // if the suggestion is nested in another one
        if (inputSug.getNestedInSuggestion() != null) {
          // get the identificator of the parent suggestion
          Integer nestedIn = null;
          if (inputSug.getNestedInSuggestion().getId() != null) {
            nestedIn = inputSug.getNestedInSuggestion().getId();
          }
          Iterator<Suggestion> suggsToDeleteIt = suggsToDelete.iterator();
          // for every suggestion that's been marked for removal
          while (suggsToDeleteIt.hasNext()) {
            Suggestion suggToDelete = suggsToDeleteIt.next();
            // ID match
            if (suggToDelete.getId() != null && nestedIn != null && suggToDelete.getId() == nestedIn) {
              // insert nested suggestion to the array for return
              toRet.add(inputSug);
              // remove the suggestion from the input array
              inputSuggsIt.remove();
              break;
            }
          }
        }
      }
    }
    return toRet;
  }  // getNestedSuggFromToDelete()
  
  /**
   * Method sets the rejected references to null and return changed suggestions.
   * 
   * @param userSuggestions list of filtered suggestions
   * @param refusedSuggestions list of refused suggestions by users
   * @return changed suggestions
   */
  private ArrayList<Suggestion> cutRefusedSuggLinks(ArrayList<Suggestion> userSuggestions, ArrayList<Suggestion> refusedSuggestions){
    ArrayList<Suggestion> changedSuggestions = new ArrayList<Suggestion>();
    if(userSuggestions != null && !userSuggestions.isEmpty() && refusedSuggestions != null && !refusedSuggestions.isEmpty()){
      Iterator<Suggestion> userSuggIt = userSuggestions.iterator();
      while(userSuggIt.hasNext()){
        Suggestion actualSugg = userSuggIt.next();
        List attrList = actualSugg.getAttributes();
        Iterator attrListIt = attrList.iterator();
        // Go trough all attributes of suggestion
        while(attrListIt.hasNext()){
          Object actualObject = (Object) attrListIt.next();
          // Search for suggestion link
          if(actualObject instanceof SugLinkedAttribute){
            SugLinkedAttribute actualLink = (SugLinkedAttribute)actualObject;
            if(actualLink.getValue() != null && actualLink.getValue() instanceof Suggestion){
              Iterator<Suggestion> refSugIterator = refusedSuggestions.iterator();
              while(refSugIterator.hasNext()){
                // Check if linked suggestion parameter points to refused suggestion
                if(((Suggestion)actualLink.getValue()).getId().equals(refSugIterator.next().getId())){
                  // Then set link to null (cut off refused suggestion)
                  actualLink.setValue(null);
                  if(!changedSuggestions.contains(actualSugg)){
                    changedSuggestions.add(actualSugg);
                  }
                }
              } // while(refSugIterator.hasNext())
            } // ... instanceof Suggestion){
          } // if(actualObject instanceof SugLinkedAttribute
          else if(actualObject instanceof SugNestedAttribute){
            SugNestedAttribute actualLink = (SugNestedAttribute)actualObject;
            if(actualLink.getValue() != null && actualLink.getValue() instanceof Suggestion){
              Iterator<Suggestion> refSugIterator = refusedSuggestions.iterator();
              while(refSugIterator.hasNext()){
                // Check if nested suggestion parameter points to refused suggestion
                if(((Suggestion)actualLink.getValue()).getId().equals(refSugIterator.next().getId())){
                  // Then set link to null (cut off refused suggestion)
                  actualLink.setValue(null);
                  if(!changedSuggestions.contains(actualSugg)){
                    changedSuggestions.add(actualSugg);
                  }
                }
              } // while(refSugIterator.hasNext())
            } // ... instanceof Suggestion){
          } // if(actualObject instanceof SugNestedAttribute
        } // while(attrListIt.hasNext())
      } // while(userSuggIt.hasNext())
    } //  if(userSuggestions != null && ... 
    return changedSuggestions;
  }  // cutRefusedSuggLinks()
  
  /**
   * Method filter list of suggestions by attributes in request.
   * 
   * @param requestInfo Informations about client request
   * @param session session where are stored attributes specified by user
   * @param userSuggestions suggestions that will be filtered
   */
  public static void filtrateSuggestions(RequestInfo requestInfo, EditorSession session, ArrayList<Suggestion> userSuggestions){
    if(requestInfo != null && requestInfo.getSession().getMinConfidence() != null && requestInfo.getSession().getMinConfidence() != 0)
      filtrateByConfidence(requestInfo.getSession().getMinConfidence(),userSuggestions);
    else
      filtrateByConfidence(MINIMAL_FILTER_CONFIDENCE,userSuggestions);
    filtrateByType(session,userSuggestions);
    filtrateByFragments(session,userSuggestions);
  }
  
  /**
   * Method filter list of suggestions by confidence.
   * 
   * @param minimalConfidence suggestions lower than this confidence will be removed from list
   * @param userSuggestions suggestions that will be filtered
   */
  private static void filtrateByConfidence(Integer minimalConfidence, ArrayList<Suggestion> userSuggestions){
    if(userSuggestions != null && !userSuggestions.isEmpty()){
      Iterator<Suggestion> suggIt = userSuggestions.iterator();
      while(suggIt.hasNext()){
        Suggestion actualSuggeston = suggIt.next();
        if(actualSuggeston.getConfidence().compareTo(minimalConfidence) < 0){
          suggIt.remove();
        }
      }
    }
  } 

  /**
   * Method go through filtered suggestions list and searches dependencies (
   * linked and nested suggestions and parents) that are in non filtered 
   * suggestion list and add them to filtered suggestion list.
   * 
   * @param filteredSuggestions Filtered suggestion list
   * @param nonFilteredSuggestion Non-filtered suggestions list
   * @return Returns resulting list with suggestions
   */
  public static ArrayList<Suggestion> addAllDependencies(ArrayList<Suggestion> filteredSuggestions, ArrayList<Suggestion> nonFilteredSuggestion) {
    ArrayList<Suggestion> open = new ArrayList<Suggestion>(nonFilteredSuggestion);
    ArrayList<Suggestion> closed = new ArrayList<Suggestion>(filteredSuggestions);
    ArrayList<Suggestion> result = new ArrayList<Suggestion>(filteredSuggestions);
    open.removeAll(result); // can't add to result something, that has been there already
    
    while(!closed.isEmpty()) {
      Suggestion actualSugg = closed.get(0);
      closed.remove(0);
      
      Iterator<Suggestion> linkedSuggIt = getAllDependencies(actualSugg).iterator();
      while(linkedSuggIt.hasNext()){
        Suggestion actualLinkedSugg = linkedSuggIt.next();
        if(open.contains(actualLinkedSugg)){
          open.remove(actualLinkedSugg);
          closed.add(actualLinkedSugg);
          result.add(actualLinkedSugg);
        }
      }
    }
    
    return result;
  }  // addAllDependencies()
  
  /**
   * Method searches in pair of suggestions and annots for pair that is specified 
   * by annotation.
   * 
   * @param pairs list where will be pair searched
   * @param annot annotation that specify pair
   */
  private AnnotSugPair getPairByAnnot(ArrayList<AnnotSugPair> pairs, Annotation annot){
    if(!pairs.isEmpty()){
      Iterator<AnnotSugPair> pairsIt = pairs.iterator();
      while(pairsIt.hasNext()){
        AnnotSugPair actualPair = pairsIt.next();
        if(actualPair.getAnnotation().getId().equals(annot.getId())){
          return actualPair;
        }
      }
    }
    
    return null;
  }
  
  /**
   * Method changes group id in annotation attribute uri.
   * 
   * @param uri old annotation type uri
   * @param groupId new group id
   */
  public static String changeGroupInUri(String uri, Integer groupId){
    if (uri == null) {
      return null;
    }
    String namesStr = uri.replace(AppBean.getBaseTypeUri(), "");
    int index = namesStr.indexOf('/');
    String withoutGroup = namesStr.substring(index);
    String result = AppBean.getBaseTypeUri() + "g" + groupId.toString() + withoutGroup;
    return result;
  }
  
  /**
   * Get annotType from list with specific URI.
   * 
   * @param list List of annotType
   * @param uri URI of AnnotType
   * @return If found - type, else null.
   */
  private static AnnotType getTypeWithSameURI(List<AnnotType> list, String uri) {
    if (list != null && !list.isEmpty()) {
      Iterator<AnnotType> newTypesIt = list.iterator();
      while (newTypesIt.hasNext()) {
        AnnotType newType = newTypesIt.next();
        // if the comparison of URIs is equal, set new type
        if (newType.getUri().equals(uri)) {
          return newType;
        }
      }
    }
    return null;
  }
  /**
   * Method check types of given suggestions, if finds unknown type create it, if 
   * finds known type change it and if type is ok (for current user group) leave 
   * type alone.
   * 
   * @param session session of user
   * @param suggestions list of suggestions which types will be checked
   * @param newTypes list of types, that was/will be created by this function
   * @param solveAtribute False - this function don't clone attribute type (although it is necessary)
   */
  public static void checkTypes(EditorSession session, ArrayList<Suggestion> suggestions, ArrayList<AnnotType> newTypes, boolean solveAtribute){
    boolean someTypesWasCloned = false;
    if(suggestions != null && !suggestions.isEmpty()){
      Iterator<Suggestion> suggIt = suggestions.iterator();
      
      // go trough all suggestions
      while(suggIt.hasNext()){
        Suggestion actualSuggestion = suggIt.next();
        AnnotType actualType = actualSuggestion.getAnnotType();
        String changedUri = changeGroupInUri(actualType.getUri(),session.getDefaultGroup().getId());

        // search also in the newTypes array
        AnnotType typeFromList = getTypeWithSameURI(newTypes, changedUri);
        if (typeFromList != null) {
          actualSuggestion.setAnnotType(typeFromList);
        } else { // type not found in the newTypes, will be created
          AnnotType typeFromDB = getAnnotTypeFromDB(changedUri);
          if(typeFromDB == null){
            // user's default group don't have this annotation type
            // make new annot type from suggestion annotation type
            AnnotType newType = cloneTypesTree(actualType, session.getDefaultGroup(), newTypes);
            actualSuggestion.setAnnotType(newType);
            someTypesWasCloned = true;
          } else {
            // user default group have this annotation type
            actualSuggestion.setAnnotType(typeFromDB);
          }
        }
      }
    }
    // clone and set annotType in cloned types
    if (solveAtribute && someTypesWasCloned) {
      int newTypesCount;
      List<AnnotType> wasProcessed = new ArrayList<AnnotType>();
      // clone types in attribute in cloned types, while any new attribute is added
      do { // do {} while (newTypesCount != newTypes.size());
        newTypesCount = newTypes.size();
        ArrayList<AnnotType> currentProcessedTypes = new ArrayList<AnnotType>(newTypes);
        for (Iterator<AnnotType> itType = currentProcessedTypes.iterator(); itType.hasNext();) {
          AnnotType type = itType.next();
          if (getTypeWithSameURI(wasProcessed, type.getUri()) != null) {
            continue;
          }
          for (Iterator<AnnotTypeAttr> itAttr = type.getAttributes().iterator(); itAttr.hasNext();) {
            AnnotTypeAttr atr = itAttr.next();
            AnnotType atrType = atr.getAttributeType();
            if (atrType != null) {
              String changedUri = changeGroupInUri(atrType.getUri(),session.getDefaultGroup().getId());
              // test, if this type has already been cloned
              if (getTypeWithSameURI(newTypes, changedUri) == null) {
                // test, if this type has already been in DB
                AnnotType typeFromDB = getAnnotTypeFromDB(changedUri);
                if(typeFromDB == null){
                  cloneTypesTree(atrType, session.getDefaultGroup(), newTypes);
                }
              }
            }
          }
          wasProcessed.add(type);
        }
      } while (newTypesCount != newTypes.size());
      // fix atribute type (atribute type has to be in user default group
      for (Iterator<AnnotType> itType = newTypes.iterator(); itType.hasNext();) {
        AnnotType type = itType.next();
        for (Iterator<AnnotTypeAttr> itAttr = type.getAttributes().iterator(); itAttr.hasNext();) {
          AnnotTypeAttr atr = itAttr.next();
          AnnotType atrType = atr.getAttributeType();
          if (atrType != null) {
            String changedUri = changeGroupInUri(atrType.getUri(),session.getDefaultGroup().getId());
            // test, if this type has already been cloned
            AnnotType clonedType = getTypeWithSameURI(newTypes, changedUri);
            if (clonedType == null) {
              // test, if this type has already been in DB
              AnnotType typeFromDB = getAnnotTypeFromDB(changedUri);
              if(typeFromDB != null){
                atr.setAttributeType(typeFromDB);
              } else {
                if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_INTERNAL_ERRORS) {
                  String msg = "Cloning of type " + changedUri + " was unsuccessful.";
                  Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
                }
              }
            } else {
              atr.setAttributeType(clonedType);
            }
          }
        }
      }
    }
  }  // checkTypes()  
    
  /************************************************************
   * UTILITY METHODS
   ************************************************************/
  
  /**
   * Method searches suggestions that will be affectedy by removing annotations.
   * 
   * @param removedAnnot list of removed annotations
   * @return list of changed suggestions
   */
  private ArrayList<Suggestion> getAffectedForDelete(ArrayList<Annotation> removedAnnot){
    ArrayList<Suggestion> changedSuggestions = new ArrayList<Suggestion>();
    if(removedAnnot != null && !removedAnnot.isEmpty()){  // if there are removed annotations
      Iterator<Annotation> annotIt = removedAnnot.iterator();
      while(annotIt.hasNext()){  // for each removed annotation
        Annotation currentAnnot = annotIt.next();
        SuggestionFeedback foundedFeedback = getFeedbackByAnnot(currentAnnot.getId());
        if(foundedFeedback != null){
          
          ArrayList<SugLinkedAttribute> attrList = getLinkedSugAttrsByLink(foundedFeedback.getSuggestionId());
          Iterator<SugLinkedAttribute> attrIt = attrList.iterator();
          while(attrIt.hasNext()){
            SugLinkedAttribute actualAttr = attrIt.next();
            changedSuggestions.add(((SugBaseAttribute)actualAttr).getRefSuggestion());
          }

          ArrayList<SugNestedAttribute> nAttrsList = getNestedSugAttrsByNested(foundedFeedback.getSuggestionId());
          Iterator<SugNestedAttribute> nAttrIt = nAttrsList.iterator();
          while(nAttrIt.hasNext()){
            SugNestedAttribute actualAttr = nAttrIt.next();
            changedSuggestions.add(((SugBaseAttribute)actualAttr).getRefSuggestion());
          }
          
          foundedFeedback.setAnnot(null);
          updateFeedback(foundedFeedback);
        } else {
          if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_INTERNAL_ERRORS) {
            String msg = "Annotation without feedback found.";
            Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
          }
        }
      }  // for each removed annotation
    }  // if there are removed annotation
    return changedSuggestions;
  }
  
  /**
   * Set low confidence to suggestions with same fragments.
   * Some suggestions may be devastated be updateSuggestionsFragments,
   * fragments of more suggestions may be same, only one of this suggestions
   * should have high confidence.
   * 
   * @param suggestions List of suggestions
   * @param changedSuggestions List of changed suggestions
   */
  private void compareFragments(ArrayList<Suggestion> suggestions, ArrayList<Suggestion> changedSuggestions) {
    ArrayList<Suggestion> processedSuggestions = new ArrayList<Suggestion>();
    for (Iterator<Suggestion> it = suggestions.iterator(); it.hasNext();) {
      Suggestion sg = it.next();
      // skip suggestion, that has been already processed
      if (processedSuggestions.contains(sg)) {
        continue;
      }
      // skip suggestion, that is nested in another suggestion
      if (sg.getNestedIn() != null) {
        continue;
      }
      // find same suggestions
      ArrayList<Suggestion> sameSuggestion = findSuggestions(suggestions, sg);
      if (sameSuggestion.size() <= 1) {
        continue;
      }
      // more suggestions are same, only one can have
      boolean oneCorrect = false;
      for (Iterator<Suggestion> itSame = sameSuggestion.iterator(); itSame.hasNext();) {
        Suggestion someSuggestion = itSame.next();
        processedSuggestions.add(someSuggestion);
        if (!oneCorrect && someSuggestion.getConfidence().compareTo(MINIMAL_FILTER_CONFIDENCE) >= 0) {
          oneCorrect = true;
        } else {
          setConfidenceToAllNested(someSuggestion, NOT_USABLE_CONFIDENCE, changedSuggestions);
        }
      }
    }
  }
  
  /**
   * Set confidence to suggestion and to its all nested suggestions.
   * 
   * @param sg Suggestion
   * @param confidence Confidence
   * @param changedSuggestions List of changed suggestions
   */
  private void setConfidenceToAllNested(Suggestion sg, Integer confidence, ArrayList<Suggestion> changedSuggestions) {
    sg.setConfidence(confidence);
    changedSuggestions.add(sg);
    List atrList = sg.getAttributes();
    if (atrList == null) {
      return;
    }
    for (Iterator itAtr = atrList.iterator(); itAtr.hasNext();) {
      Object obj = itAtr.next();
      if (!(obj instanceof SugNestedAttribute)) {
        continue;
      }
      SugNestedAttribute atr = (SugNestedAttribute) obj;
      if (atr.getNestedSuggestion() != null) {
        setConfidenceToAllNested(atr.getNestedSuggestion(), confidence, changedSuggestions);
      }
    }
  }
  
  /**
   * Method goes through all suggestions and their fragments and try to update 
   * them. If some suggestion have bad fragment sets that suggestions minimal
   * confidence.
   * 
   * @param suggestions old suggestions to update
   * @param request informations about client request
   * @param changedSugg Array for changed suggestions
   */
  private ArrayList<Suggestion> updateSuggestionsFragments(ArrayList<Suggestion> suggestions, RequestInfo request, ArrayList<Suggestion> changedSugg){
    if(suggestions != null && !suggestions.isEmpty()){
      
      // go trough all suggestions
      Iterator<Suggestion> suggIt = suggestions.iterator();
      while(suggIt.hasNext()){
        
        Suggestion actualSuggestion = suggIt.next();
        // if suggestion have any fragment
        if(actualSuggestion.getFragments() != null){
          // indicate if any suggestion fragment was changed
          boolean isChanged;
          // update suggestion fragments
          isChanged = updateSuggestion(actualSuggestion,request);
          // check updated suggestion fragment if there is some bad fragment
          Iterator<SuggestionFragment> fragIt = actualSuggestion.getFragments().iterator();
          while(fragIt.hasNext()){
            SuggestionFragment actualFragment = fragIt.next();
            // if some fragment is bad, set suggestion minimal confidence
            if(!actualFragment.getIsGood()){
              actualSuggestion.setConfidence(NOT_USABLE_CONFIDENCE);
            }
          }
          
          if(isChanged){
            changedSugg.add(actualSuggestion);
          }
        }
      }
    }
    
    return changedSugg;
  }  // updateSuggestionsFragments()
  
  /**
   * Method adds a new suggestions to the old suggestions. Method uses these 
   * rules:
   *  - If the new suggestion is found in the old suggestions then is increased 
   *    its confidence
   *  - If the new suggestion isn't found in the old suggestions then is added 
   *    to old suggestions
   *  - If there is suggestion in old suggestion that isn't in new suggestion 
   *    then is decreased its confidence
   * 
   * @param newSuggestions new suggestions from ner
   * @param oldSuggestions existing suggestions for document that are in DB
   * @param addedSugg there will be new annotations saved
   * @param changedSugg there will be changed annotations saved
   */
  private void insertNewSuggestions(ArrayList<Suggestion> newSuggestions, ArrayList<Suggestion> oldSuggestions, ArrayList<Suggestion> addedSugg, ArrayList<Suggestion> changedSugg){
    if(newSuggestions != null && !newSuggestions.isEmpty() && oldSuggestions != null){
      
      if(!oldSuggestions.isEmpty()){
        // is important to find corresponding suggestions
        Iterator<Suggestion> newSugIt = newSuggestions.iterator();
        while(newSugIt.hasNext()){
          Suggestion actualNewSuggestion = newSugIt.next();
          // try to find current new suggestion in old suggestions (by comparing contents)
          Suggestion findedSugg = findSuggestion(oldSuggestions,actualNewSuggestion);
          // test if corresponding suggestion is founded
          if(findedSugg != null){
            // suggestion survived update
            Suggestion finded = findSuggInArray(findedSugg.getId(),changedSugg);
            if(finded != null){
              if (!finded.contentEquals(actualNewSuggestion, false, true)) {
                finded.setConfidence(NOT_USABLE_CONFIDENCE);
                actualNewSuggestion.setConfidence(INITIAL_CONFIDENCE);
                addedSugg.add(actualNewSuggestion);
              }
            }else{
              //findedSugg.setConfidence(MINIMAL_CONFIDENCE);
              //changedSugg.add(findedSugg);
            }
          }else{
            // suggestion is new
            actualNewSuggestion.setConfidence(INITIAL_CONFIDENCE);
            addedSugg.add(actualNewSuggestion);
          }
        }
        
        // try to find suggestions that aren't in new suggestions but they are in old, they are abandoned
        //
        // Idea of this code is good but effect is wrong as NER is not able to find all usable 
        // suggestions and we can not determine the source of suggestion. So it is better to comment 
        // it out now.
        /*Iterator<Suggestion> oldSugIt = oldSuggestions.iterator();
        while(oldSugIt.hasNext()){
          Suggestion actualOldSugg = oldSugIt.next();
          if(findSuggestion(newSuggestions,actualOldSugg) == null && findSuggestion(changedSugg,actualOldSugg) == null){
            // suggestion is abandoned 
            actualOldSugg.setConfidence(MINIMAL_CONFIDENCE);
            changedSugg.add(actualOldSugg);
          }
        }*/
        
      }else{
        // nothing to compare, newSuggestions will be inserted to oldSuggestions
        Iterator<Suggestion> newSugIt = newSuggestions.iterator();
        while(newSugIt.hasNext()){
          Suggestion sug = newSugIt.next();
          if (sug != null && sug.getConfidence() == null) {
            sug.setConfidence(INITIAL_CONFIDENCE);
          }
        }
        addedSugg.addAll(newSuggestions);
      }
    }
  }  // insertNewSuggestions()
  
  /**
   * Method finds suggestion by id in suggestions array.
   * 
   * @param id id of searched suggestion
   * @param list Array to search in
   */
  private Suggestion findSuggInArray(Integer id, ArrayList<Suggestion> list){
    if(list != null && !list.isEmpty()){
      Iterator<Suggestion> listIt = list.iterator();
      while(listIt.hasNext()){
        Suggestion actualSuggestion = listIt.next();
        if(actualSuggestion.getId() != null && actualSuggestion.getId().equals(id)){
          return actualSuggestion;
        }
      }
    }
    return null;
  }
  
  /**
   * Updates fragments in suggestion. (Method copied from CoreFuncMOdule)
   *
   * @param suggestion Suggestion to update
   * @param request Informations about client request
   * @return If suggestion was updated, returns true, if no update needed or
   * error occurred, returns false
   */
  public static boolean updateSuggestion(Suggestion suggestion, RequestInfo request) {
    
    // Provider of matchers for finding fragments (exact matches only)
    MatcherProvider exactMatcherProvider;
    // Provider of matchers for finding fragments (nearest matches only)
    MatcherProvider nearestMatcherProvider;
    // Provider of matchers for finding fragments (sequence iterator)
    MatcherProvider sequenceMatcherProvider;

    exactMatcherProvider = new MatcherProvider();
    nearestMatcherProvider = new MatcherProvider();
    sequenceMatcherProvider = new MatcherProvider();

    // create nearest matcher
    nearestMatcherProvider.add(new Matcher(new BidirectionallyUnNestNodeIterator(), new Comparator(new ExactMethod(), Comparator.TraversingMethod.CHARACTER_BIDIRECTIONALLY)), 0);
    nearestMatcherProvider.add(new Matcher(new BidirectionallyUnNestNodeIterator(), new Comparator(new LevenshteinMethod(0.8), Comparator.TraversingMethod.WORD)), 1);
    nearestMatcherProvider.add(new Matcher(new BidirectionallyUnNestNodeIterator(), new Comparator(new LevenshteinMethod(0.8), Comparator.TraversingMethod.FIRST_AND_LAST_WORD_THEN_LETTER, 2.0, 2.0)), 2);

    // create sequence mathcer
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new ExactMethod(), Comparator.TraversingMethod.CHARACTER_BIDIRECTIONALLY)), 0);
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new LevenshteinMethod(0.8), Comparator.TraversingMethod.FIRST_AND_LAST_WORD_THEN_LETTER, 2.0, 2.0)), 1);
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new LevenshteinMethod(0.5), Comparator.TraversingMethod.WORD)), 2);

    // create exact matcher
    exactMatcherProvider.add(new Matcher(new XPathNodeIterator(), new Comparator(new ExactMethod(), Comparator.TraversingMethod.STATIC)), 0);

    if (suggestion.getFragments().size() < 1) {
      // if annotation haven't fragments, it can't be updated
      return false;
    }

    // get synchronized document
    Document parsedDoc = request.getSession().getParsedSyncDocument();
    if (parsedDoc == null) {
      //make message about error and throw it
      //"Bad document for update of the suggestion."
      return false;  // updating is not possible
    }

    int badFragCnt = 0;  // total count of bad fragments
    int nowOrphFragCnt = 0;  // count of just orphaned fragments
    int updatedFragCnt = 0;  // count of just updated fragments
    Iterator<SuggestionFragment> fragIt = suggestion.getFragments().iterator();
    while (fragIt.hasNext()) {  // for each suggestion fragment
      SuggestionFragment fragment = fragIt.next();
      try {
        // find fragment
        UpdatableFragment uf = exactMatcherProvider.match(parsedDoc, fragment.toUpdatableFragment());
        if (uf != null) {  // if fragment was found
          if (fragment.getIsGood() == false) {  // if fragment was marked as bad
            fragment.setIsGood(true);  // mark as good
            updatedFragCnt++;  // fragment was now updated
          }
        } else {  // if fragment was not found
          // find fragment in document with full matching capability
          //ArrayList <UpdatableFragment> ufl = matcherProvider.matchAllIncrementally(parsedDoc, fragment.toUpdatableFragment());
          ArrayList<UpdatableFragment> ufl = new ArrayList<UpdatableFragment>();
          // find nearest fragment
          UpdatableFragment ufTemp = nearestMatcherProvider.matchInClosestNode(parsedDoc, fragment.toUpdatableFragment());
          if (ufTemp != null) {
            ufl.add(ufTemp);
          } else {
            // find fragment with sequence iterator
            ufl = sequenceMatcherProvider.matchAllIncrementally(parsedDoc, fragment.toUpdatableFragment());
          }
          if (ufl.size() == 1) {  // if one fragment was found
            // get fragment (index unknown)
            Iterator<UpdatableFragment> ufIt = ufl.iterator();
            uf = ufIt.next();
            fragment.updateWithUpdatableFragment(uf);  // update fragment in suggestion
            updatedFragCnt++;
            if (fragment.getIsGood() == false) {  // if fragment was marked as bad
              fragment.setIsGood(true);  // mark as good
            }
          } else if (ufl.isEmpty()) {  // if no fragment found
            if (fragment.getIsGood() == true) {  // if fragment was not bad
              fragment.setIsGood(false);  // mark as bad
              nowOrphFragCnt++;  // fragment was now orphaned
            }
            badFragCnt++;
          } else {  // if searching is ambiguous
            // select the best fragment
            int minBadness = Integer.MAX_VALUE;  // minimal badness
            UpdatableFragment minBFR = null;  // fragment with minimal L.D.
            String origPath = fragment.getPath();  // original XPath
            String origText = fragment.getAnnotatedText();  // original annotated text
            int origOffset = fragment.getOffset();  // original offset
            for (Iterator<UpdatableFragment> uflIt = ufl.iterator(); uflIt.hasNext();) {
              UpdatableFragment ufr = uflIt.next();
              int lD = Util.levenshtein(origPath, ufr.getXPathString());
              int lDT = Util.levenshtein(origText, ufr.getText());
              int offDistance = Math.abs(origOffset - ufr.getOffset());
              int badness = 10000 * lD + 1000 * lDT + offDistance;
              if (badness < minBadness) {
                minBadness = badness;
                minBFR = ufr;
              }
            }
            uf = minBFR;
            fragment.updateWithUpdatableFragment(uf);  // update fragment in annotation
            updatedFragCnt++;
            if (fragment.getIsGood() == false) {  // if fragment was marked as bad
              fragment.setIsGood(true);  // mark as good
            }
          }  // if searching is ambiguous
        }  // if fragment was not found
      } catch (XPathExpressionException ex) {  // bad XPath in fragment
        if (fragment.getIsGood() == true) {  // if fragment was not bad
          fragment.setIsGood(false);  // mark as bad
          nowOrphFragCnt++;  // fragment was now orphaned
        }
        badFragCnt++;
      } catch (Exception e) {
        if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_INTERNAL_ERRORS) {
          String msg = "Update of suugestion fragments failed.";
          Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg, e);
        }
        badFragCnt++;
        break;
      }
    }  // for each annotated fragment

    if (badFragCnt == suggestion.getFragments().size() && nowOrphFragCnt > 0) {
      // if annotation was just orphaned
    } else if (nowOrphFragCnt > 0) {
      // if suggestion was just partially orphaned (bad fragments found)
    }
    if (nowOrphFragCnt > 0 || updatedFragCnt > 0) {
      // if new bad fragments found or some fragments was updated
      return true;
    }
    return false;  // nothing updated (no update needed)
  }  // updateSuggestion()
  
  /**
   * Method finds suggestion in the suggestion list by its content.
   * 
   * @param suggestions suggestion list where method will search
   * @param suggestion suggestion which will be seek
   * @return founded suggestion
   */
  private Suggestion findSuggestion(ArrayList<Suggestion> suggestions, Suggestion suggestion){
    Iterator<Suggestion> newSugIt = suggestions.iterator();
    while(newSugIt.hasNext()){
      Suggestion actualSugg = newSugIt.next();
      if(actualSugg.contentEquals(suggestion, false, true)){
        return actualSugg;
      }
    }
    
    return null;
  }
  
    /**
   * Method finds suggestions in the suggestion list by its content.
   * Suggestions, not suggestion.
   * 
   * @param suggestions suggestion list where method will search
   * @param suggestion suggestion which will be seek
   * @return founded suggestions in list.
   */
  private ArrayList<Suggestion> findSuggestions(ArrayList<Suggestion> suggestions, Suggestion suggestion){
    ArrayList<Suggestion> findedSuggestions = new ArrayList<Suggestion>();
    Iterator<Suggestion> newSugIt = suggestions.iterator();
    while(newSugIt.hasNext()){
      Suggestion actualSugg = newSugIt.next();
      if(actualSugg.contentEquals(suggestion, false, true)){
        findedSuggestions.add(actualSugg);
      }
    }
    
    return findedSuggestions;
  }
  
  /**
   * Method converts list of fragments to list of SuggestionFragments.
   * 
   * @param fragments list of fragments
   * @param reference Suggestion to which new fragment should belong
   * @return list of converted fragments
   */
  private static ArrayList<SuggestionFragment> fragToSuggFrag(ArrayList<Fragment> fragments, Suggestion reference){
    ArrayList<SuggestionFragment> result = new ArrayList<SuggestionFragment>();
    if(fragments != null && !fragments.isEmpty()){
      // go trough all fragments
      Iterator<Fragment> fragIt = fragments.iterator();
      while(fragIt.hasNext()){
        Fragment currentFragment = fragIt.next();
        // create new suggestion fragment and set its prarameters 
        SuggestionFragment newFragment = new SuggestionFragment();
        newFragment.setAnnotatedText(currentFragment.getAnnotatedText());
        newFragment.setIsGood(currentFragment.getIsGood());
        newFragment.setLength(currentFragment.getLength());
        newFragment.setOffset(currentFragment.getOffset());
        newFragment.setPath(currentFragment.getPath());
        newFragment.setRefSuggestion(reference);
        result.add(newFragment);
      }
    }
    
    return result;
  }  // fragToSuggFrag()
  
  /**
   * Method filter list of suggestions by fragments.
   * 
   * @param session session where are stored fragments specified by user
   * @param userSuggestions suggestions that will be filtered
   */
  private static void filtrateByFragments(EditorSession session, ArrayList<Suggestion> userSuggestions){
    if(userSuggestions != null && !userSuggestions.isEmpty() 
            && session.getSuggestionsFr() != null 
            && !session.getSuggestionsFr().isEmpty()){
      // get document
      Document doc = session.getParsedSyncDocument();
      // linearize fragments for suggestions defined by user
      Integer offset = session.getSuggestionsFr().get(0).getOffset();
      Integer length = session.getSuggestionsFr().get(0).getLength();
      
      // turn suggestions off
      if(length != null && length == 0) {
        session.getSuggestionsFr().clear();
        userSuggestions.clear();
        return;
      }
      // request for the whole document
      else if (offset == null && length == null) {
        return;
      }
      ArrayList<SuggestionFragment> linFragments = getLinearized(doc,fragToSuggFrag(session.getSuggestionsFr(),null));
      if(linFragments != null && !linFragments.isEmpty()){
        
        // gotrough all suggestions 
        Iterator<Suggestion> suggIterator = userSuggestions.iterator();
        while(suggIterator.hasNext()){
          Suggestion actualSuggestion = suggIterator.next();
          
          // linearize suggestion fragment
          Iterator<SuggestionFragment> linFragIt = linFragments.iterator();
          // try to find by user defined fragent of document where suggestion fits
          while(linFragIt.hasNext()){
            if(!isInFragment(doc,actualSuggestion,linFragIt.next())){
              suggIterator.remove();
            }
          }
        }
      }
      
      if(session.getSuggestionsFr() != null) {
        session.getSuggestionsFr().clear();
      }
    } else  if(userSuggestions != null && !userSuggestions.isEmpty()){
      userSuggestions.clear();
    }
  }  // filtrateByFragments()
  
  /**
   * Method filter list of suggestions by their type.
   * 
   * @param session session from user where is stored type that specifies suggestions
   * @param userSuggestions suggestions that will be filtered
   */
  private static void filtrateByType(EditorSession session, ArrayList<Suggestion> userSuggestions){
    if(userSuggestions != null && !userSuggestions.isEmpty()){
      // Get type of annotations requested by user
      ArrayList<String> requestedTypeUris = session.getSuggestionsTypes();
      if(requestedTypeUris != null && !requestedTypeUris.isEmpty()){
        if (!(requestedTypeUris.size() == 1 && requestedTypeUris.get(0) == null)) {
          // go trough all suggestions
          Iterator<Suggestion> suggIterator = userSuggestions.iterator();
          while (suggIterator.hasNext()) {
            boolean fits = false;  
            /* As URIs may come from different groups for e.g.
             * http://localhost:8080/Annotations/types/g1/Person/Coreference_to_Person
             * and 
             * http://localhost:8080/Annotations/types/g2/Person
             * it's needed to cut off group prefix from both URIs and after that comparison
             * can be used
             */
            Suggestion actualSuggestion = suggIterator.next();
            String actualUri = actualSuggestion.getAnnotType().getUri();
            
            /* Cut off prefix from URI */
            actualUri = actualUri.replace(AppBean.getBaseTypeUri(), "");
            actualUri = actualUri.substring(actualUri.indexOf("/")+1);
            
            Iterator<String> urisIterator = requestedTypeUris.iterator();
            while (urisIterator.hasNext()) {
              String suggType = urisIterator.next();
              /* Cut off prefix from URI */
              suggType = suggType.replace(AppBean.getBaseTypeUri(), "");
              suggType= suggType.substring(suggType.indexOf("/")+1);
              
              if ((actualUri.startsWith(suggType) && actualUri.length() > suggType.length() && actualUri.charAt(suggType.length()) == '/') 
                      || actualUri.equals(suggType)) {
                fits = true;
              }
            }

            // if suggestion is not requested type
            if (!fits) {
              // delete it
              suggIterator.remove();
            }
          }
        }
      }
    }  // if(userSuggestions != null...
  }  // filtrateByType()
  
  /**
   * Method make from list of structured fragments list of linearized fragments.
   * 
   * @param doc document that fragments belongs
   * @param fragments list of structured fragments
   * @return list of linearized fragments
   */
  public static ArrayList<SuggestionFragment> getLinearized(Document doc, ArrayList<SuggestionFragment> fragments){
    ArrayList<ArrayList<SuggestionFragment>> notConverted = new ArrayList<ArrayList<SuggestionFragment>>();
    ArrayList<ArrayList<SuggestionFragment>> comFragments = new ArrayList<ArrayList<SuggestionFragment>>();
    
    comFragments.add(fragments);
    
    ArrayList<SuggestionFragment> linearizedList;
    linearizedList = Linearizer.fragmentsToLinSugFragments(comFragments,doc, notConverted, true);
    
    if (!notConverted.isEmpty()) {
      // notConverted is an array of arrays, so we need to go deeper...
      Iterator<ArrayList<SuggestionFragment>> notCIT = notConverted.iterator();
      while (notCIT.hasNext()) {
        ArrayList<SuggestionFragment> notC = notCIT.next();
        // one array contains fragments within
        if (!notC.isEmpty()) {
          if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
            String msg = "Some fragments was not linearized successfully.";
            Logger.getLogger(SuggestionManager.class.getName()).log(Level.WARNING, msg);
          }
        }
      }
    }
    
    return linearizedList;
  }
  
  /**
   * Method check if suggestion is in linear document fragment.
   * 
   * @param doc document that suggestion and fragment belongs
   * @param sugg suggestion that will be tested
   * @param linFragment linearized document fragment that will be tested
   */
  private static boolean isInFragment(Document doc, Suggestion sugg, SuggestionFragment linFragment){
    Iterator<SuggestionFragment> linSuggFragmentsIt;   
    
    if(sugg.getFragmentsAL().isEmpty() || sugg.getFragmentsAL().get(0) == null){
        return false;
    }
    
    try{
      linSuggFragmentsIt = getLinearized(doc, sugg.getFragmentsAL()).iterator();
    }catch(RuntimeException ex){
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = ex + "Can't convert fragments of suggestion: " + sugg.toString();
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
      }
      
      return false;
    }
    
    // get position of reference fragment content in linearized document
    int fragStart = linFragment.getOffset().intValue();
    int fragEnd = fragStart + linFragment.getLength().intValue();
    
    while(linSuggFragmentsIt.hasNext()){
      SuggestionFragment actualSuggFragment = linSuggFragmentsIt.next();

      // get position of current suggestion fragment content in linearized document
      int sugFragStart = actualSuggFragment.getOffset().intValue();
      int sugFragEnd = sugFragStart + actualSuggFragment.getLength().intValue();

      if(fragStart <= sugFragStart && fragEnd >= sugFragStart){
        // suggestion fragment starts in reference fragment
        return true;
      }else if(fragStart <= sugFragEnd && fragEnd >= sugFragEnd){
        // suggestion fragment ends in reference fragment
        return true;
      }
    }
    return false;
  }  // isInFragment()
  
  /**
   * Method finds all linked suggestions of suggestions and return them as ArrayList.
   * 
   * @param suggestion suggestion for which will be linked suggestions searched
   * @return list of founded linked suggestions
   */
  private ArrayList<Suggestion> getAllLinkedSugg(Suggestion suggestion){
    ArrayList<Suggestion> retArray = new ArrayList<Suggestion>(); 
    if(suggestion != null){
      
      // go trough all attributes of suggestion
      List attrObjects = suggestion.getAttributes();
      if(attrObjects != null && !attrObjects.isEmpty()){
        Iterator attrIt = attrObjects.iterator();
        while(attrIt.hasNext()){
          Object actualObject = (Object) attrIt.next();
          
          // try find linked suggestion attributes
          if(actualObject instanceof SugLinkedAttribute){
            // if find linked suggestion attribute
            SugLinkedAttribute currentLink = (SugLinkedAttribute)actualObject;
            if(currentLink.getValue() != null && currentLink.getValue() instanceof Suggestion){
              // add it to return array
              retArray.add(((Suggestion)currentLink.getValue()));
            }
          }
        }
      }
    }
    
    return retArray;
  }  // getAllLinkedSugg()
  
  /**
   * Method clone attributes annot type without group, id, uri and attributes.
   * 
   * @param referenceType annotation type that will be clone
   * @return new annotation type cloned from referenced type
   */
  private static AnnotType cloneType(AnnotType referenceType){
    AnnotType newType = new AnnotType();
    newType.setComment(referenceType.getComment());
    newType.setGroup(null);
    newType.setName(referenceType.getName());
    newType.setRestrictedAtt(referenceType.getRestrictedAtt());
    newType.setUriInOntology(referenceType.getUriInOntology());
    return newType;
  }
  
  /**
   * Method clone attributes reference annot type and copy them to new type.
   * 
   * @param referenceType attributes of this annotation type will be clone
   * @param newType cloned atributes will be copied to this annotation type
   */
  private static void cloneTypeAttributes(AnnotType referenceType, AnnotType newType){
    ArrayList<AnnotTypeAttr> clonedAttr = new ArrayList<AnnotTypeAttr>();
    if(referenceType.getAttributes() != null){
      // go trough all annotation type attributes
      Iterator<AnnotTypeAttr> attrIt = referenceType.getAttributes().iterator();
      while(attrIt.hasNext()){
        // make list of new annotation type attributes
        AnnotTypeAttr newAttr = new AnnotTypeAttr(attrIt.next());
        newAttr.setAnnotationType(newType);
        clonedAttr.add(newAttr);
      }
    }
    newType.setAttributes(clonedAttr);
  }

  /**
   * Method tests if array of annotation types contains type with specific uri.
   * 
   * @param types array of types
   * @param uri searched type uri
   * @return true if type is found
   */
  private static boolean isAnnotTypeInArray(ArrayList<AnnotType> types, String uri){
    if(types != null && !types.isEmpty()){
      Iterator<AnnotType> typesIt = types.iterator();
      while(typesIt.hasNext()){
        if(typesIt.next().getUri().equals(uri)){
          return true;
        }
      }
    }
    
    return false;
  }
  
  /**
   * Method clone given type and all his ancestors tree if they aren't stored in 
   * DB under given user group.
   * 
   * @param currentType type of otrer group for clone
   * @param currentGroup current group of clonned type
   * @param clonedTypes list of cloned types, this list is filled in this function, out of this function is this list processed
   * @return cloned currentType
   */
  private static AnnotType cloneTypesTree(AnnotType currentType, UserGroup currentGroup, ArrayList<AnnotType> clonedTypes){
    String changedURI = changeGroupInUri(currentType.getUri(),currentGroup.getId());
            
    // process same type again - have to stop recursion
    for (Iterator<AnnotType> it = clonedTypes.iterator(); it.hasNext();) {
      AnnotType type = it.next();
      if (type.getUri().equals(changedURI)) {
        return type;
      }
    }
    
    // clone current type (without ancestors)
    AnnotType clonedType = cloneType(currentType);
    clonedType.setUri(changedURI);
    clonedType.setGroup(currentGroup);
    // clone attributes of current ancestor
    cloneTypeAttributes(currentType, clonedType);
      
    // don't add same already added ancestors
    if(!isAnnotTypeInArray(clonedTypes,clonedType.getUri())){
      clonedTypes.add(clonedType);
    }
      
    ArrayList<AnnotType> ancestorsOfType = currentType.getAncestorTypesAL();
    if(ancestorsOfType != null && !ancestorsOfType.isEmpty()){
      // go trough all ancestors
      Iterator<AnnotType> ancestorsOfTypeIt = ancestorsOfType.iterator();
      while(ancestorsOfTypeIt.hasNext()){
        
        // try to find if current ancestor is already in DB for current user group
        AnnotType currentAncestorType = ancestorsOfTypeIt.next();
        String changedUri = changeGroupInUri(currentAncestorType.getUri(),currentGroup.getId());
        AnnotType dbType = getAnnotTypeFromDB(changedUri);
        if(dbType == null){
          // this ancestor isn't in DB for current user group
          // recursively call ths method 
          dbType = cloneTypesTree(currentAncestorType,currentGroup,clonedTypes);
          // if current ancestor have any ancestor
          if(dbType != null){
            // add all tree of ancestors to array of ancestors to save
            if (!clonedTypes.contains(dbType)) {
              clonedTypes.add(dbType);
            }
          }else{
            // this part of tree is already added
            continue;
          }  
        }

        // if it is first add of ancestor, set him as main ancestor
        if(clonedType.getAncestorType() == null){
          clonedType.setAncestorType(dbType);
        }

        // add ancestor to list of ancestors
        clonedType.addAncestorType(dbType);
      }
    }
    
    return clonedType;
  }
  

  /**
   * If ancestor of type of annotation is not exists, but should and name
   * is probably composed of path in type tree, this method breakdowns path,
   * sets name properly and generate ancestors.
   * New names are generated from URI.
   * Ancestors are stored into informations about client request.
   *
   * @param type Type of annotation to check and breakdown
   * @param requestInfo Informations about client request
   * @param newTypes List of newly added types
   */
  public static ArrayList<AnnotType> breakdownType(AnnotType type, RequestInfo requestInfo, ArrayList<AnnotType> newTypes) {
    ArrayList<AnnotType> changedTypes = new ArrayList<AnnotType>();
    // get user group to which this type belongs
    UserGroup group = type.getGroup();
    // get string with name of type and names of his ancestors
    String namesStr = type.getUri().replace(AppBean.getBaseTypeUri(), "");
    // get array with name of type and names of his ancestors
    String[] namesArr = namesStr.split("/");
    ArrayList<String> names = new ArrayList<String>(Arrays.asList(namesArr));
    int nSize = names.size();  // compute number of names
    if (nSize > 2) {  // if there is more then name of type and id of group
      // get name of type and remove it from list
      String typeName = names.remove(nSize - 1);
      nSize--;
      type.setName(typeName);  // set proper name to type
      AnnotType nextType = type;  // set actually checked type to checked type
      while (nSize > 1) {  // create ancestor types (last name is group name)
        // get name of actual (next to now generated) type
        String nextName = nextType.getName();
        // get name of ancestor (previous in hierarchy)
        String prevName = names.remove(nSize - 1);
        nSize--;
        // get URI of actual type
        String nUri = nextType.getUri();
        // get URI of ancestor type
        // prevUri = uri of next without "/" and name of next
        String prevUri = nUri.substring(0, nUri.lastIndexOf("/"));
        // flag represents a successful search for a type in the newTypes array
        boolean foundInNewTypes = false;
        
        // query database for ancestor type
        Object[] params = new Object[2];
        params[0] = "uri";
        params[1] = prevUri;
        List tList = AppBean.getPersistenceManager().queryDB("AnnotType.findByUri", params);
        if (tList != null && !tList.isEmpty()) {  // if ancestor was found
          AnnotType prevType = (AnnotType) tList.get(0);
          nextType.setAncestorType(prevType);  // set ancestor properly
          nextType.addAncestorType(prevType);
          break;  // end of checking (all other is in database)
        }
        else if (newTypes != null && !newTypes.isEmpty()) {
          Iterator<AnnotType> newTypesIt = newTypes.iterator();
          while (newTypesIt.hasNext()) {
            AnnotType newType = newTypesIt.next();
            if (newType.getUri().equals(prevUri)) {
              nextType.setAncestorType(newType);  // set ancestor properly
              nextType.addAncestorType(newType);
              foundInNewTypes = true;
              break;
            }
          }
        }

        // if not found in DB nor newTypes, create ancestor type
        if (!foundInNewTypes) {
          AnnotType newType = new AnnotType(prevUri, prevName, null, group);
          nextType.setAncestorType(newType);  // set ancestor of actual type to new ancestor
          nextType.addAncestorType(newType);
          changedTypes.add(newType);  // add new ancestor type to flier
          nextType = newType;  // set ancestor as actually checked type
        }
      }  // create ancestor types (last name is group name)
    }  // if there is more then name of type and id of group
    return changedTypes;
  }  // breakdownType()
  
  /**
   * Persist types of annotations to DB
   *
   * @param types Types of annotations to persist
   * @return In case of success returns false, true otherwise
   */
  public static boolean persistTypes(ArrayList<AnnotType> types) {
    EntityManager em = AppBean.getPersistenceManager().getEM();
    EntityTransaction transaction = em.getTransaction();
    try {
      transaction.begin();

      ArrayList<AnnotType> alreadyStored = new ArrayList<AnnotType>();
      
      // create list without attributes and ansestors and map with original types
      ArrayList<AnnotType> nTWA = new ArrayList<AnnotType>();
      Map<String, AnnotType> nTMap = new HashMap<String, AnnotType>();
      for (Iterator<AnnotType> ntIt = types.iterator(); ntIt.hasNext();) {
        AnnotType type = ntIt.next();
        if (type.getId() != null) {
          alreadyStored.add(type);
          continue;
        }
        AnnotType nT = new AnnotType(type.getUri(), type.getName(), null, type.getGroup());
        nT.setRestrictedAtt(type.getRestrictedAtt());
        nT.setUriInOntology(type.getUriInOntology());
        nT.setComment(type.getComment());
        nTWA.add(nT);
        nTMap.put(type.getUri(), type);
      }

      // merge types without attributes and create map with merged types
      ArrayList<AnnotType> mergedTypes = new ArrayList<AnnotType>();
      Map<String, AnnotType> mTypesMap = new HashMap<String, AnnotType>();
      for (Iterator<AnnotType> nTWAIt = nTWA.iterator(); nTWAIt.hasNext();) {
        AnnotType type = nTWAIt.next();
        if (type.getId() != null) {
          continue;
        }
        type = em.merge(type);
        mergedTypes.add(type);
        mTypesMap.put(type.getUri(), type);
      }

      // fix, add and merge attributes and ancestors
      for (Iterator<AnnotType> mIt = mergedTypes.iterator(); mIt.hasNext();) {
        AnnotType mType = mIt.next();
        AnnotType oType = nTMap.get(mType.getUri());  // get original type
        
        // skip type, that is stored in DB
        if (oType == null || mType == null || mType.getId() != null || oType.getId() != null){
          continue;
        }
        
        // attributes
        List<AnnotTypeAttr> aList = oType.getAttributes();
        for (Iterator<AnnotTypeAttr> aIt = aList.iterator(); aIt.hasNext();) {
          AnnotTypeAttr aTA = aIt.next();
          AnnotType aTAAT = aTA.getAttributeType();
          if (aTAAT != null) {  // if it is structured attribute
            // replace type with merged one
            AnnotType rType = mTypesMap.get(aTAAT.getUri());
            if (rType == null) {  // it is not new type, it is in database
              rType = queryType(aTAAT.getUri(), em);
            }
            aTA.setAttributeType(rType);
          }
          // fix back reference to annotation type
          aTA.setAnnotationType(mTypesMap.get(aTA.getAnnotationType().getUri()));
        }
        mType.setAttributes(new ArrayList<AnnotTypeAttr>(aList));

        // primary ancestor
        if (oType.getAncestorType() != null) {
          // replace type with merged one
          AnnotType rType = mTypesMap.get(oType.getAncestorType().getUri());
          if (rType == null) {  // it is not new type, it is in database
            rType = queryType(oType.getAncestorType().getUri(), em);
          }
          mType.setAncestorType(rType);
        }

        // ancestors
        ArrayList<AnnotType> newAncestors = new ArrayList<AnnotType>();
        if (oType.getAncestorTypes() != null) {
          for (Iterator<AnnotType> ancIt = oType.getAncestorTypes().iterator(); ancIt.hasNext();) {
            AnnotType ancType = ancIt.next();
            AnnotType mAnc = mTypesMap.get(ancType.getUri());
            if (mAnc == null) {  // it is not new type, it is in database
              mAnc = queryType(ancType.getUri(), em);
            }
            // skip type, that is stored in DB
            if (mAnc != null && (ancType.getId() != null || mAnc.getId() != null)) {
              continue;
            }
            newAncestors.add(mAnc);
          }
        }
        mType.setAncestorTypes(newAncestors);

        em.merge(mType);
      }  // fix, add and merge attributes and ancestors

      em.flush();
      transaction.commit();
      mergedTypes.addAll(alreadyStored);
      types.clear();
      types.addAll(mergedTypes);
      return false;
    } catch (Exception e) {
      transaction.rollback();
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = "Persisting of types of annotations failed.";
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg, e);
      }
      return true;
    } finally {
      em.close();
    }
  }  // persistTypes()

  /************************************************************
   * DATABASE MANIPULATION METHODS
   ************************************************************/

  /**
   * Query database for type of annotation
   *
   * @param uri URI of type of annotation
   * @param em Entity manager
   * @return Returns type of annotation with given URI
   * @throws RuntimeException If type of annotation was not found, throws exception
   */
  public static AnnotType queryType(String uri, EntityManager em) throws RuntimeException {
    AnnotType annotType = null;
    Object[] params = new Object[2];
    params[0] = "uri";
    params[1] = uri;
    Query q = em.createNamedQuery("AnnotType.findByUri");
    for (int p = 0; p < params.length; p = p + 2) {
      q.setParameter((String) params[p], params[p + 1]);
    }
    List aList = q.getResultList();
    if (aList != null && !aList.isEmpty()) {  // if type was found
      annotType = (AnnotType) aList.get(0);
    } else {  // if type was not found (merge failed)
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_SERVER_ERRORS) {
        String msg = "New type of annotation was not found.";
        Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
      }
      throw new RuntimeException("New type not found!");
    }
    return annotType;
  }  // queryType()
  
  /**
   * Method loads suggestions specified by document and confidence from the database.
   * 
   * @param documentId document id that specify suggestions
   * @param minimalConfidence minimal confidence of suggestions
   * @return list of querried suggestions
   */
  private ArrayList<Suggestion> getSuggestionsForAlternatives(Integer documentId, int minimalConfidence){
    ArrayList<Suggestion> result = new ArrayList<Suggestion>();
    
    // get suggestions for document from database
    Object[] params = {"sourceDocId",documentId,"minCon",minimalConfidence};
    @SuppressWarnings("unchecked")
    List<Suggestion> suggestionList = AppBean.getPersistenceManager().queryDB("Suggestion.findForAlternatives",params);

    processSugAttributes(suggestionList, result);
    return result;
  }
  
  /**
   * Method loads suggestions specified by document from the database.
   * 
   * @param documentId document id that specify suggestions
   * @return list of querried suggestions
   */
  public static ArrayList<Suggestion> getSuggestionsFromDB(Integer documentId){
    ArrayList<Suggestion> result = new ArrayList<Suggestion>();
    
    // get suggestions for document from database
    Object[] params = {"sourceDocId",documentId};
    @SuppressWarnings("unchecked")
    List<Suggestion> suggestionList = AppBean.getPersistenceManager().queryDB("Suggestion.findBySourceDocId",params);

    processSugAttributes(suggestionList, result);
    return result;
  }
  
  /**
   * Process suggestion attributes
   * 
   * @param suggestionList Original list of suggestions
   * @param result New list of suggestions
   */
  public static void processSugAttributes(List<Suggestion> suggestionList, ArrayList<Suggestion> result) {
    if(suggestionList != null && !suggestionList.isEmpty()){
      Iterator<Suggestion> suggestionListIt = suggestionList.iterator();
      while(suggestionListIt.hasNext()){
        Suggestion actualSug = suggestionListIt.next();
        List<SugBaseAttribute> originalAttributes = actualSug.getAttributes();
        Iterator<SugBaseAttribute> attrIt = originalAttributes.iterator();
        ArrayList<SugBaseAttribute> newAttributes = new ArrayList<SugBaseAttribute>();
        while(attrIt.hasNext()){
          SugBaseAttribute attr = attrIt.next();
          if(!SugAttributeManager.hasAttributeRightInstance(attr)){
            SugBaseAttribute newAttr = SugAttributeManager.changeAttributeInstance(attr);
            newAttributes.add(newAttr);
            attrIt.remove();
          }
        }
        if(!newAttributes.isEmpty()){
          originalAttributes.addAll(newAttributes);
          actualSug.setAttributes(new ArrayList<SugBaseAttribute>(originalAttributes));
        }
      }
      result.addAll(suggestionList);
    }
  }
  /**
   * Method loads annotation type specified by group and uri from the database.
   * 
   * @param groupId id of group of querried annotation type
   * @param uri uri of querried annotation type
   * @return annotation type that fits by groupId and uri
   */
  private static AnnotType getAnnotTypeFromDB(Integer groupId, String uri){
    Object[] params = {"uri",uri,"groupId",groupId};
    @SuppressWarnings("unchecked")
    List<AnnotType> types = AppBean.getPersistenceManager().queryDB("AnnotType.findByUriAndGroup",params);
    if(types == null || types.isEmpty()){
      return null;
    }
    
    return types.get(0);
  }
  
  /**
   * Method loads annotation type specified by uri from the database.
   * 
   * @param uri Uri of queried annotation type
   * @return Returns annotation type that fits by uri
   */
  public static AnnotType getAnnotTypeFromDB(String uri){
    Object[] params = {"uri",uri};
    @SuppressWarnings("unchecked")
    List<AnnotType> types = AppBean.getPersistenceManager().queryDB("AnnotType.findByUri",params);
    if(types == null || types.isEmpty()){
      return null;
    }
    
    return types.get(0);
  }
  
  /**
   * Method loads suggestion attributes specified by suggestion link.
   * 
   * @param suggId id of suggestion
   * @return list of founded linked suggestion attributes
   */
  private ArrayList<SugLinkedAttribute> getLinkedSugAttrsByLink(Integer suggId){
    ArrayList<SugLinkedAttribute> result = new ArrayList<SugLinkedAttribute>();
    Object[] params = {"linked",suggId};
    @SuppressWarnings("unchecked")
    List<SugLinkedAttribute> attr = AppBean.getPersistenceManager().queryDB("SuggestionAttribute.findByLinked",params);
    if(attr != null && !attr.isEmpty()){
      result.addAll(attr);
    }
    
    return result;
  }
  
  /**
   * Method loads suggestion attributes specified by nested suggestion.
   * 
   * @param suggId id of nested suggestion
   * @return list of founded linked suggestion attributes
   */
  private ArrayList<SugNestedAttribute> getNestedSugAttrsByNested(Integer suggId){
    ArrayList<SugNestedAttribute> result = new ArrayList<SugNestedAttribute>();
    Object[] params = {"nested",suggId};
    @SuppressWarnings("unchecked")
    List<SugNestedAttribute> attr = AppBean.getPersistenceManager().queryDB("SuggestionAttribute.findByNested",params);
    if(attr != null && !attr.isEmpty()){
      result.addAll(attr);
    }
    
    return result;
  }
  
  /**
   * Method store new suggestion feedback to DB.
   * 
   * @param feedback suggestion feedback that will be stored
   */
  private static void persistFeedback(SuggestionFeedback feedback){
    if(feedback != null){
      if(AppBean.getPersistenceManager().persistEntity(feedback)){
        // Error, input path is null
        if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_ALL_ERRORS) {
          String msg = "Can't save feedback to DB. Object: \n" + feedback.toString();
          Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
        }
      }
    }
  }
  
  /**
   * Method updates suggestion feedback in DB.
   * 
   * @param feedback suggestion feedback that will be update
   */
  private static void updateFeedback(SuggestionFeedback feedback){
    if(feedback != null){
      SuggestionFeedback savedSuggestionF = (SuggestionFeedback)AppBean.getPersistenceManager().saveEntityChanges(feedback);
      if(savedSuggestionF == null){
        // Error, input path is null
        if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_ALL_ERRORS) {
          String msg = "Can't save feedback to DB. Object: \n" + feedback.toString();
          Logger.getLogger(SuggestionManager.class.getName()).log(Level.SEVERE, msg);
        }
      }
    }
  }
  
  /**
   * Method loads suggestion feedback specified by group and document from the database.
   * 
   * @param documentId document id that specify suggestion feedback
   * @param groupId group id that specify suggestion feedback
   */
  public static ArrayList<SuggestionFeedback> getFeedbackFromDB(Integer documentId, Integer groupId){
    ArrayList<SuggestionFeedback> result = new ArrayList<SuggestionFeedback>();
    
    Object[] params = {"groupId",groupId,"docId",documentId};
    @SuppressWarnings("unchecked")
    List<SuggestionFeedback> feedbackList = AppBean.getPersistenceManager().queryDB("SuggestionFeedback.findByDoc",params);
    if(feedbackList != null && !feedbackList.isEmpty()){
      result.addAll(feedbackList);
    }
    
    return result;
  }
  
  /**
   * Method loads suggestion positive feedback specified by annotation.
   * 
   * @param annotId annotation id that specify suggestion feedback
   * @return array list with founded negative suggestions feedback
   */
  public static SuggestionFeedback getFeedbackByAnnot(Integer annotId){
    Object[] params = {"annotationId",annotId};
    @SuppressWarnings("unchecked")
    List<SuggestionFeedback> feedbackList = AppBean.getPersistenceManager().queryDB("SuggestionFeedback.findByAnnot",params);
    if(feedbackList != null && !feedbackList.isEmpty()){
      return feedbackList.get(0);
    }
    
    return null;
  }
  
  /**
   * Method loads negative suggestion feedback specified by group and document from 
   * the database.
   * 
   * @param documentId document id that specify suggestion feedback
   * @param groupId group id that specify suggestion feedback
   * @return array list with founded negative suggestions feedback
   */
  private ArrayList<SuggestionFeedback> getNegFeedbackFromDB(Integer documentId, Integer groupId){
    ArrayList<SuggestionFeedback> result = new ArrayList<SuggestionFeedback>();
    
    Object[] params = {"groupId",groupId,"docId",documentId};
    @SuppressWarnings("unchecked")
    List<SuggestionFeedback> feedbackList = AppBean.getPersistenceManager().queryDB("SuggestionFeedback.findByDocNeg",params);
    if(feedbackList != null && !feedbackList.isEmpty()){
      result.addAll(feedbackList);
    }
    
    return result;
  }
  
  /**
   * Method loads suggestion feedback specified by suggestion ID and group ID.
   * 
   * @param suggestionId Id of suggestion for which we are searching suggestion feedback
   * @param groupId ID of group to which feedback belongs
   * @return Returns querried suggestion feedback or null if feedback isn't found
   */
  private static SuggestionFeedback getFeedbackSFromDB(Integer suggestionId, Integer groupId){
    Object[] params = {"suggestionId",suggestionId,"groupId",groupId};
    @SuppressWarnings("unchecked")
    List<SuggestionFeedback> feedbackList = AppBean.getPersistenceManager().queryDB("SuggestionFeedback.findBySuggestionId",params);
    if(feedbackList != null && !feedbackList.isEmpty()){
      return feedbackList.get(0);
    }
    return null;
  }
  
  /**
   * Method loads suggestion specified by id from the database.
   * 
   * @param id id that specify suggestions
   * @return querried suggestion or null if suggestion isn't found
   */
  public static Suggestion getSuggestionFromDB(Integer id){
    Object[] params = {"id",id};
    @SuppressWarnings("unchecked")
    List<Suggestion> suggestionList = AppBean.getPersistenceManager().queryDB("Suggestion.findById",params);
    if(suggestionList != null && !suggestionList.isEmpty()){
      return suggestionList.get(0);
    }
    
    return null;
  }
  
  /************************************************************
   * DOCUMENT LOCK METHODS
   ************************************************************/
  
   /**
   * Method locks the document suggestions for change.
   * 
   * @param documentId id of document
   */
  public void lockDocForChange(Integer documentId){
    while(true){
      if(AppBean.getLockMaster().getSuggestionLock(documentId).lockForChange()){
        break;
      }
    }
  }
  
  /**
   * Method unlocks the document suggestions for change.
   * 
   * @param documentId id of document
   */
  public void unlockDocForChange(Integer documentId){
    AppBean.getLockMaster().getSuggestionLock(documentId).unlockForChange();
  }
  
  /**
   * Method test if is there request for document update.
   * 
   * @param request informations about client request
   * @return true if is ther request for document update
   */
  private boolean isDocUpdate(RequestInfo request){
    if((request.getTextModifications() != null && !request.getTextModifications().isEmpty())
       || (requestInfo.getSyncDocument() != null 
           && (getSuggestionsFromDB(requestInfo.getSyncDocument().getId()).isEmpty() 
               || !requestInfo.getFlier().getTextModifications().isEmpty()))
       || requestInfo.getResyncDocContent() != null){
      return true;
    }
    return false;
  }
  
  /**
   * Method test if is there request for suggestions.
   * 
   * @param request informations about client request
   * @return true if is ther request for suggestions
   */
  private boolean isSuggestionsRequest(RequestInfo request){
    if((request.isSuggFragmentChanged() && request.getSyncDocument() == null)){
      return true;
    }
    return false;
  }
  
  /**
   * Method test if is there request with positive feedback.
   * 
   * @param request informations about client request
   * @return true if is ther request with positive feedback
   */
  private boolean isPositiveFeedback(RequestInfo request){
    if(request.getConfirmedSuggestions() != null && !request.getConfirmedSuggestions().isEmpty()){
      return true;
    }
    
    return false;
  }
  
  /**
   * Method test if is there request with negative feedback.
   * 
   * @param request informations about client request
   * @return true if is ther request with negative feedback
   */
  private boolean isNegativeFeedback(RequestInfo request){
    if(request.getRefusedSuggestions() != null && !request.getRefusedSuggestions().isEmpty()){
      return true;
    }
    
    return false;
  }
  
  /**
   * Method test if is there request with changed annotations.
   * 
   * @param request informations about client request
   * @return true if is ther request with changed annotations
   */
  private boolean isAnnotChange(RequestInfo request){
    if(request.getFlier().getEditedAnnotations() != null 
    && !request.getFlier().getEditedAnnotations().isEmpty()){
      return true;
    }
    
    return false;
  }
  
  /**
   * Method test if is there request with removed annotations.
   * 
   * @param request informations about client request
   * @return true if is ther request with removed annotations
   */
  private boolean isAnnotDelete(RequestInfo request){
    if(request.getFlier().getRemovedAnnotations() != null 
    && !request.getFlier().getRemovedAnnotations().isEmpty()){
      return true;
    }
    
    return false;
  }
  
  /**
   * Method test if is there request with added annotations.
   * 
   * @param request informations about client request
   * @return true if is ther request with added annotations
   */
  private boolean isAnnotAdded(RequestInfo request){
    if(request.getFlier().getAddedAnnotations() != null 
    && !request.getFlier().getAddedAnnotations().isEmpty()){
      return true;
    }
    
    return false;
  }
  
  /**
   * Creates escaped version of the provided textual data for use in JSON format 
   * for SEC API.
   * Method expects SEC API to store strings enclosed in double quote. 
   * That's why double quote character needs to be escaped and single quote not.
   * 
   * @param source string with characters to be escaped
   * @return Returns escaped veersion of string
   */
  private static String EscapeForSecApi(String source) {
    String result = source;
    if (result != null) {
      result = result.replace("\\", "\\\\");
      result = result.replace("\"", "\\\"");
      result = result.replace("\n", " ");
      result = result.replace("\t", " ");
    }
    return result;
  } // EscapeForSecApi()
  
}  // class SuggestionManager