Linearizer.java

Go to the documentation of this file.

/*
 * Project: Server for annotations sharing
 * Author: Ing. Jaroslav Dytrych idytrych@fit.vutbr.cz
 * File: Linearizer.java
 * Description: This class contains utility functions for document linearization.
 */

/**
 * @file Linearizer.java
 *
 * @brief Utility functions for document linearization.
 */
package cz.vutbr.fit.knot.annotations.comet;

import cz.vutbr.fit.knot.annotations.app.TextModification;
import cz.vutbr.fit.knot.annotations.entity.Fragment;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.MatcherProvider;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.XPathHelper;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.SuggestionFragment;
import cz.vutbr.fit.knot.annotations.modules.suggestionManager.alternative.AlternativeFragment;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import javax.xml.parsers.ParserConfigurationException;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.traversal.DocumentTraversal;
import org.w3c.dom.traversal.NodeFilter;
import org.w3c.dom.traversal.NodeIterator;
import org.xml.sax.SAXException;

/**
 * This class contains utility functions for document linearization.
 *
 * @brief Utility functions for document linearization.
 * @author idytrych
 */
public class Linearizer {

  /**
   * Matcher provider for linearizing of fragments
   */
  private static MatcherProvider matcherProvider;

  /**
   * Convert list of linearized fragments to common fragments.
   * For each linearized fragment more common fragments can be created.
   *
   * @param linFragments List of linearized fragments. Fragments must be sorted
   *                     by offset!
   * @param doc Document in which fragments are
   * @param badFragments List to which bad fragments will be stored
   * @return Returns list of lists of common fragments
   */
  public static ArrayList<ArrayList<Fragment>> linFragmentsToFragments(ArrayList<Fragment> linFragments,
                                                                       Document doc,
                                                                       ArrayList<ArrayList<Fragment>> badFragments) {
    ArrayList<ArrayList<Fragment>> retFragments = new ArrayList<ArrayList<Fragment>>();
    ArrayList<Integer> processedLengths = new ArrayList<Integer>();
    int linLength = linFragments.size();  // number of linearized fragments
    for (int i = 0; i < linLength; i++) {  // create lists for results
      retFragments.add(new ArrayList<Fragment>());
      badFragments.add(new ArrayList<Fragment>());
      processedLengths.add(0);
    }
    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      int convertedLin = 0;  // index of last fully converted fragment
      Node currentNode = nIter.nextNode();
      int newlineOffsetCompensation = 0; // offset increment for new line

      while (currentNode != null && convertedLin < linLength) {
             // while there is next node and there is something to convert
        nContent = currentNode.getNodeValue();
        nEndOffset = nStartOffset + nContent.length();
        int i = convertedLin;
        Fragment lFr = null;
        do {
          lFr = linFragments.get(i);
          // modify the offset according to the new line character count
          lFr.setOffset(lFr.getOffset() - newlineOffsetCompensation);
          int lFrEnd = lFr.getOffset() + lFr.getLength();
          if (lFr.getOffset() < nStartOffset && lFrEnd < nStartOffset) {
              // if fragment is already converted
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() >= nStartOffset && lFrEnd <= nEndOffset) {
                     // if fragment is fully contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = lFr.getOffset() - nStartOffset;
            String content = nContent.substring(offset, offset + lFr.getLength());
            Fragment f = new Fragment(path, offset, lFr.getLength(), content, null);
            if (lFr.getAnnotatedText().equals(content)) {
              // if fragment content is matching
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, lFr.getLength());
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() >= nStartOffset && lFr.getOffset() < nEndOffset
                     && lFrEnd > nEndOffset) {  // if start of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = lFr.getOffset() - nStartOffset;
            Integer length = nContent.length() - offset;
            String content = nContent.substring(offset, nContent.length());
            Fragment f = new Fragment(path, offset, length, content, null);
            int lFTL = lFr.getAnnotatedText().length();
            if (length <= lFTL && lFr.getAnnotatedText().substring(0, length).equals(content)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, length);
          } else if (lFr.getOffset() < nStartOffset && lFrEnd > nStartOffset
                     && lFrEnd <= nEndOffset) {  // if end of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = 0;
            Integer length = lFrEnd - nStartOffset;
            String content = nContent.substring(0, lFrEnd - nStartOffset);
            int lFTL = lFr.getAnnotatedText().length();
            Fragment f = new Fragment(path, offset, length, content, null);
            if (processedLengths.get(i) < lFTL
                && lFr.getAnnotatedText().substring(processedLengths.get(i), lFTL).equals(content)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, processedLengths.get(i) + length);
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() < nStartOffset && lFrEnd > nEndOffset) {
                     // if part of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = 0;
            Integer length = nContent.length();
            int lFTL = lFr.getAnnotatedText().length();
            int lFTE = processedLengths.get(i) + length;
            Fragment f = new Fragment(path, offset, length, nContent, null);
            if (processedLengths.get(i) < lFTL && lFTE <= lFTL &&
                lFr.getAnnotatedText().substring(processedLengths.get(i), lFTE).equals(nContent)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, processedLengths.get(i) + length);
          } else {
            // fragment has no matching text in the current node
          }
          // reverse the offset modification
          lFr.setOffset(lFr.getOffset() + newlineOffsetCompensation);
          
          i++;
        } while (i < linLength && lFr.getOffset() < nEndOffset);

        nStartOffset += nContent.length();  // start offset of next node
        currentNode = nIter.nextNode();  // move to next node of document
        newlineOffsetCompensation += 1;  // increment the compensation factor
      }  // while there is next node and there is something to convert
    }  // if document element is presented

    return retFragments;
  }  // linFragmentsToFragments()

  /**
   * Convert list of linearized fragments to common fragments.
   * For each linearized fragment more common fragments can be created.
   *
   * @param linFragments List of linearized fragments. Fragments must be sorted
   *                     by offset!
   * @param doc Document in which fragments are
   * @param badFragments List to which bad fragments will be stored
   * @return Returns list of lists of common fragments
   */
  public static ArrayList<ArrayList<SuggestionFragment>> linSugFragmentsToFragments(ArrayList<SuggestionFragment> linFragments,
                                                                       Document doc,
                                                                       ArrayList<ArrayList<SuggestionFragment>> badFragments) {
    ArrayList<ArrayList<SuggestionFragment>> retFragments = new ArrayList<ArrayList<SuggestionFragment>>();
    ArrayList<Integer> processedLengths = new ArrayList<Integer>();
    int linLength = linFragments.size();  // number of linearized fragments
    for (int i = 0; i < linLength; i++) {  // create lists for results
      retFragments.add(new ArrayList<SuggestionFragment>());
      badFragments.add(new ArrayList<SuggestionFragment>());
      processedLengths.add(0);
    }
    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      int convertedLin = 0;  // index of last fully converted fragment
      Node currentNode = nIter.nextNode();
      int newlineOffsetCompensation = 0; // offset increment for new line

      while (currentNode != null && convertedLin < linLength) {
             // while there is next node and there is something to convert
        nContent = currentNode.getNodeValue();
        
        // if the node is a non breaking space, we can say that it doesn't
        // contain any suggestion, so we can speed up the evaluation
        if (nContent.replaceAll("[\\s\\u00A0]+$", "").contentEquals("")) {
          currentNode = nIter.nextNode();  // move to next node of document
          continue;
        }
        
        nEndOffset = nStartOffset + nContent.length();
        int i = convertedLin;
        SuggestionFragment lFr = null;
        do {
          lFr = linFragments.get(i);
          // modify the offset according to the new line character count
          lFr.setOffset(lFr.getOffset() - newlineOffsetCompensation);
          int lFrEnd = lFr.getOffset() + lFr.getLength();
          if (lFr.getOffset() < nStartOffset && lFrEnd < nStartOffset) {
              // if fragment is already converted
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() >= nStartOffset && lFrEnd <= nEndOffset) {
                     // if fragment is fully contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = lFr.getOffset() - nStartOffset;
            String content = nContent.substring(offset, offset + lFr.getLength());
            SuggestionFragment f = new SuggestionFragment(path, offset, lFr.getLength(), content, null);
            if (lFr.getAnnotatedText().equals(content)) {
              // if fragment content is matching
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, lFr.getLength());
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() >= nStartOffset && lFr.getOffset() < nEndOffset
                     && lFrEnd > nEndOffset) {  // if start of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = lFr.getOffset() - nStartOffset;
            Integer length = nContent.length() - offset;
            String content = nContent.substring(offset, nContent.length());
            SuggestionFragment f = new SuggestionFragment(path, offset, length, content, null);
            int lFTL = lFr.getAnnotatedText().length();
            if (length <= lFTL && lFr.getAnnotatedText().substring(0, length).equals(content)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, length);
          } else if (lFr.getOffset() < nStartOffset && lFrEnd > nStartOffset
                     && lFrEnd <= nEndOffset) {  // if end of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = 0;
            Integer length = lFrEnd - nStartOffset;
            String content = nContent.substring(0, lFrEnd - nStartOffset);
            int lFTL = lFr.getAnnotatedText().length();
            SuggestionFragment f = new SuggestionFragment(path, offset, length, content, null);
            if (processedLengths.get(i) < lFTL
                && lFr.getAnnotatedText().substring(processedLengths.get(i), lFTL).equals(content)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, processedLengths.get(i) + length);
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() < nStartOffset && lFrEnd > nEndOffset) {
                     // if part of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = 0;
            Integer length = nContent.length();
            int lFTL = lFr.getAnnotatedText().length();
            int lFTE = processedLengths.get(i) + length;
            SuggestionFragment f = new SuggestionFragment(path, offset, length, nContent, null);
            if (processedLengths.get(i) < lFTL && lFTE <= lFTL &&
                lFr.getAnnotatedText().substring(processedLengths.get(i), lFTE).equals(nContent)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, processedLengths.get(i) + length);
          } else {
            // fragment has no matching text in the current node
          }
          // reverse the offset modification
          lFr.setOffset(lFr.getOffset() + newlineOffsetCompensation);
          
          i++;
        } while (i < linLength && lFr.getOffset() < nEndOffset);

        nStartOffset += nContent.length();  // start offset of next node
        currentNode = nIter.nextNode();  // move to next node of document
        newlineOffsetCompensation += 1;  // increment the compensation factor
      }  // while there is next node and there is something to convert
    }  // if document element is presented

    return retFragments;
  }  // linSugFragmentsToFragments()

  /**
   * Convert list of linearized fragments to common fragments.
   * For each linearized fragment more common fragments can be created.
   *
   * @param linFragments List of linearized fragments. Fragments must be sorted
   *                     by offset!
   * @param doc Document in which fragments are
   * @param badFragments List to which bad fragments will be stored
   * @return Returns list of lists of common fragments
   */
  public static ArrayList<ArrayList<AlternativeFragment>> linAltFragmentsToFragments(ArrayList<AlternativeFragment> linFragments,
                                                                       Document doc,
                                                                       ArrayList<ArrayList<AlternativeFragment>> badFragments) {
    ArrayList<ArrayList<AlternativeFragment>> retFragments = new ArrayList<ArrayList<AlternativeFragment>>();
    ArrayList<Integer> processedLengths = new ArrayList<Integer>();
    int linLength = linFragments.size();  // number of linearized fragments
    for (int i = 0; i < linLength; i++) {  // create lists for results
      retFragments.add(new ArrayList<AlternativeFragment>());
      badFragments.add(new ArrayList<AlternativeFragment>());
      processedLengths.add(0);
    }
    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      int convertedLin = 0;  // index of last fully converted fragment
      Node currentNode = nIter.nextNode();
      int newlineOffsetCompensation = 0; // offset increment for new line

      while (currentNode != null && convertedLin < linLength) {
             // while there is next node and there is something to convert
        nContent = currentNode.getNodeValue();
        
        // if the node is a non breaking space, we can say that it doesn't
        // contain any suggestion, so we can speed up the evaluation
        if (nContent.replaceAll("[\\s\\u00A0]+$", "").contentEquals("")) {
          currentNode = nIter.nextNode();  // move to next node of document
          continue;
        }
        
        nEndOffset = nStartOffset + nContent.length();
        int i = convertedLin;
        AlternativeFragment lFr = null;
        do {
          lFr = linFragments.get(i);
          // modify the offset according to the new line character count
          lFr.setOffset(lFr.getOffset() - newlineOffsetCompensation);
          int lFrEnd = lFr.getOffset() + lFr.getLength();
          if (lFr.getOffset() < nStartOffset && lFrEnd < nStartOffset) {
              // if fragment is already converted
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() >= nStartOffset && lFrEnd <= nEndOffset) {
                     // if fragment is fully contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = lFr.getOffset() - nStartOffset;
            String content = nContent.substring(offset, offset + lFr.getLength());
            AlternativeFragment f = new AlternativeFragment(path, offset, lFr.getLength(), content, null);
            if (lFr.getAnnotatedText().equals(content)) {
              // if fragment content is matching
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, lFr.getLength());
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() >= nStartOffset && lFr.getOffset() < nEndOffset
                     && lFrEnd > nEndOffset) {  // if start of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = lFr.getOffset() - nStartOffset;
            Integer length = nContent.length() - offset;
            String content = nContent.substring(offset, nContent.length());
            AlternativeFragment f = new AlternativeFragment(path, offset, length, content, null);
            int lFTL = lFr.getAnnotatedText().length();
            if (length <= lFTL && lFr.getAnnotatedText().substring(0, length).equals(content)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, length);
          } else if (lFr.getOffset() < nStartOffset && lFrEnd > nStartOffset
                     && lFrEnd <= nEndOffset) {  // if end of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = 0;
            Integer length = lFrEnd - nStartOffset;
            String content = nContent.substring(0, lFrEnd - nStartOffset);
            int lFTL = lFr.getAnnotatedText().length();
            AlternativeFragment f = new AlternativeFragment(path, offset, length, content, null);
            if (processedLengths.get(i) < lFTL
                && lFr.getAnnotatedText().substring(processedLengths.get(i), lFTL).equals(content)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, processedLengths.get(i) + length);
            if (i == convertedLin) {  // if offset in list can be moved
              convertedLin++;
            }
          } else if (lFr.getOffset() < nStartOffset && lFrEnd > nEndOffset) {
                     // if part of fragment is contained
            String path = XPathHelper.XPathStringOfNode(currentNode);
            Integer offset = 0;
            Integer length = nContent.length();
            int lFTL = lFr.getAnnotatedText().length();
            int lFTE = processedLengths.get(i) + length;
            AlternativeFragment f = new AlternativeFragment(path, offset, length, nContent, null);
            if (processedLengths.get(i) < lFTL && lFTE <= lFTL &&
                lFr.getAnnotatedText().substring(processedLengths.get(i), lFTE).equals(nContent)) {
              retFragments.get(i).add(f);  // add fragment to the appropriate list
            } else {
              badFragments.get(i).add(f);  // add fragment to the appropriate list
            }
            processedLengths.set(i, processedLengths.get(i) + length);
          } else {
            // fragment has no matching text in the current node
          }
          // reverse the offset modification
          lFr.setOffset(lFr.getOffset() + newlineOffsetCompensation);
          
          i++;
        } while (i < linLength && lFr.getOffset() < nEndOffset);

        nStartOffset += nContent.length();  // start offset of next node
        currentNode = nIter.nextNode();  // move to next node of document
        newlineOffsetCompensation += 1;  // increment the compensation factor
      }  // while there is next node and there is something to convert
    }  // if document element is presented

    return retFragments;
  }  // linAltFragmentsToFragments()
  
  /**
   * Linearizes document to string
   *
   * @param doc Document to linearize
   * @return Returns linearized document in string
   */
  public static String linearizeDocument(Document doc) {
    StringBuilder linDoc = new StringBuilder();
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Node currentNode = nIter.nextNode();
      // flag represents new line offset compensation
      boolean compensateNewline = false;
      while (currentNode != null) {
        // the node containg only non breaking space character, therefore there
        // can't be any annotation it it, skip it

        if (currentNode.getNodeValue() == null || currentNode.getNodeValue().replaceAll("[\\s\\u00A0]+$", "").contentEquals("")) {
          currentNode = nIter.nextNode();  // move to next node of document
          continue;
        }
    // append the space character after the last character of the node text data
    // only if the value node is not a non breaking space character
        if (compensateNewline) {
    // this solution resolves the problem with joining words delimited
    // by the <br> tag
          linDoc.append(" ");
        }
        linDoc.append(currentNode.getNodeValue());
        currentNode = nIter.nextNode();  // move to next node of document
        compensateNewline = true;
      }
    }  // if document element is presented
    return linDoc.toString();
  }  // linearizeDocument()

  /**
   * Convert list of common fragments to linearized fragments.
   * For more common fragments one linearized fragment can be created.
   *
   * @param comFragments List of common fragments.
   * @param doc Document in which fragments are
   * @param notConverted List for fragments, which was not converted
   * @param addSpaces If true, it will add space after each node (offset will be incremented by 1)
   * @return Returns list of linearized fragments
   */
  public static ArrayList<SuggestionFragment> fragmentsToLinSugFragments(ArrayList<ArrayList<SuggestionFragment>> comFragments,
                                                            Document doc, ArrayList<ArrayList<SuggestionFragment>> notConverted,
                                                            boolean addSpaces) {

    // create copy of array from which fragments will be removed and count fragments
    int fragCount = 0;
    int numOfAnnots = comFragments.size();

    ArrayList<ArrayList<SuggestionFragment>> comFrCopy = new ArrayList<ArrayList<SuggestionFragment>>();
    // array of linearized fragments that will be returned by this method
    ArrayList<SuggestionFragment> retFragments = new ArrayList<SuggestionFragment>(numOfAnnots);
    // auxiliary array containing fragments from previous document node
    ArrayList<SuggestionFragment> partialFragments = new ArrayList<SuggestionFragment>(numOfAnnots);
    // auxiliary array containing number of spaces to add before start offset
    ArrayList<Integer> numsOfSpacesSO = new ArrayList<Integer>(numOfAnnots);
    // auxiliary array containing number of spaces to add into length
    ArrayList<Integer> numsOfSpacesL = new ArrayList<Integer>(numOfAnnots);
    
    for (int i = 0; i < numOfAnnots; i++) {
      ArrayList<SuggestionFragment> cFr = comFragments.get(i);
      ArrayList<SuggestionFragment> aFragments = new ArrayList<SuggestionFragment>();
      notConverted.add(new ArrayList<SuggestionFragment>());
      comFrCopy.add(aFragments);
      for (Iterator<SuggestionFragment> aFrIt = cFr.iterator(); aFrIt.hasNext();) {
        SuggestionFragment fr = aFrIt.next();
        aFragments.add(fr);
        fragCount++;
      }
      // these two arrays should be initialized here, because their size is
      // equal to the number of annotations
      retFragments.add(null);
      partialFragments.add(null);
      numsOfSpacesSO.add(null);
      numsOfSpacesL.add(null);
    }

    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      Node currentNode = nIter.nextNode();
      int nodeCounter = 0;
      String path = "";
      while (currentNode != null && fragCount > 0) {
        // while there is next node and there is something to convert
        nContent = currentNode.getNodeValue();
        int nContentL = nContent.length();
        nEndOffset = nStartOffset + nContentL;
        path = XPathHelper.XPathStringOfNode(currentNode);
        
        for (int i = 0; i < numOfAnnots; i++) {
          // for all sets of fragments
          ArrayList<SuggestionFragment> aFragments = comFrCopy.get(i);
          for (Iterator<SuggestionFragment> frIt = aFragments.iterator(); frIt.hasNext();) {
            // for all fragments for one annotation
            SuggestionFragment fr = frIt.next();
            if (fr.getIsGood() == false) {
              notConverted.get(i).add(fr);  // fragment can not be converted
              frIt.remove();  // fragment was processed
              fragCount--;
              continue;
            }
            if (fr.getPath().equals(path)) {  // if fragment is in the node
              SuggestionFragment partFrag = partialFragments.get(i);
              int frEndOffset;
              if (fr.getOffset() != null) {
                frEndOffset = fr.getOffset() + fr.getLength();
              }
              else {
                frEndOffset = fr.getLength();
              }
              if (frEndOffset > nContentL) {  // content of fragment is too long
                notConverted.get(i).add(fr);
                frIt.remove();
                fragCount--;
                continue;
              }   else if (fr.getAnnotatedText() != null && !nContent.substring(fr.getOffset(), frEndOffset).equals(fr.getAnnotatedText())) {
                // fr.getAnnotatedText() returns null when linearizing suggestion fragments
                // content is not matching
                notConverted.get(i).add(fr);
                frIt.remove();
                fragCount--;
                continue;
              }
              if (partFrag == null && frEndOffset < nContentL) {
                // whole linearized fragment in node
                if (fr.getOffset() == null) {
                  // When user requests fragment from the start of a node editor sends offset as null, which causes NullPointerException
                  fr.setOffset(0);
                }
                numsOfSpacesSO.set(i,nodeCounter);  // set number of spaces before start offset for this fragment
                numsOfSpacesL.set(i,0);  // set number of spaces inside length for this fragment
                SuggestionFragment linFragment = new SuggestionFragment("", nStartOffset + fr.getOffset(),
                                                    fr.getLength(), fr.getAnnotatedText(),
                                                    fr.getRefSuggestion());
                frIt.remove();  // fragment was successfully processed
                fragCount--;
                retFragments.set(i, linFragment);  // set result
              } else if (partFrag == null && frEndOffset == nContentL) {

                // start of linearized fragment in node
                if (fr.getOffset() == null) {
                  fr.setOffset(0);
                }
                numsOfSpacesSO.set(i,nodeCounter);  // set number of spaces before start offset for this fragment
                numsOfSpacesL.set(i,0);  // set number of spaces inside length for this fragment
                SuggestionFragment linFragment = new SuggestionFragment("", nStartOffset + fr.getOffset(),
                                                    fr.getLength(), fr.getAnnotatedText(),
                                                    fr.getRefSuggestion());

                partialFragments.set(i, linFragment);
                frIt.remove();  // fragment was successfully processed
                fragCount--;
              } else if (partFrag != null && frEndOffset < nContentL) {
                // end of linearized fragment in node
                SuggestionFragment linFragment = partialFragments.get(i);
                if (fr.getAnnotatedText() != null) {
                  linFragment.setAnnotatedText(linFragment.getAnnotatedText() + fr.getAnnotatedText());
                }
                linFragment.setLength(linFragment.getLength() + fr.getLength());
                retFragments.set(i, linFragment);
                partialFragments.set(i, null);
                if (numsOfSpacesL.get(i) != null) {  // add space after previous node
                  numsOfSpacesL.set(i,numsOfSpacesL.get(i) + 1);
                }
                frIt.remove();
                fragCount--;
              } else if (partFrag != null && frEndOffset == nContentL) {
                // part of linearized fragment in node
                SuggestionFragment linFragment = partialFragments.get(i);
                if (fr.getAnnotatedText() != null) {
                  linFragment.setAnnotatedText(linFragment.getAnnotatedText() + fr.getAnnotatedText());
                }
                linFragment.setLength(linFragment.getLength() + fr.getLength());
                if (numsOfSpacesL.get(i) != null) {  // add space after previous node
                  numsOfSpacesL.set(i,numsOfSpacesL.get(i) + 1);
                }
                frIt.remove();
                fragCount--;
              } else {
                // error state
                notConverted.get(i).add(fr);  // fragment can not be converted
                frIt.remove();  // fragment was processed
                fragCount--;
                continue;
              }
            }  // if fragment is in the node
          }  // for all fragments for one annotation
        }  // for all sets of fragments


        nStartOffset += nContent.length();  // start offset of next node
        currentNode = nIter.nextNode();  // move to next node of document
        while (currentNode != null && (currentNode.getNodeValue() == null || currentNode.getNodeValue().replaceAll("[\\s\\u00A0]+$", "").contentEquals(""))) {
          currentNode = nIter.nextNode();  // move to next node of document
        }
        nodeCounter++;
      }  // while there is next node and there is something to convert
    }  // if document element is presented

    for (int i = 0; i < numOfAnnots; i++) {
      // for all sets of fragments
      ArrayList<SuggestionFragment> aFragments = comFrCopy.get(i);
      // move rest of fragments
      for (Iterator<SuggestionFragment> aFrIt = aFragments.iterator(); aFrIt.hasNext();) {
        SuggestionFragment fr = aFrIt.next();
        notConverted.get(i).add(fr);
        aFrIt.remove();
        fragCount--;
      }

      // finish processing of partial fragment
      if (partialFragments.get(i) != null) {
        retFragments.set(i, partialFragments.get(i));  // set result
      }
    }  // for all sets of fragments

    if (addSpaces) {  // if spaces should be added (offsets incremented)
      for (int i = 0; i < numOfAnnots; i++) {  // for each converted fragment
        SuggestionFragment fr = retFragments.get(i);
        if (fr == null) {  // fragment was not converted successfully
          continue;
        }
        // increment offset
        Integer spaces = numsOfSpacesSO.get(i);
        if (spaces == null) {
          spaces = 0;
        }
        fr.setOffset(fr.getOffset() + spaces);
        // increment length
        spaces = numsOfSpacesL.get(i);
        if (spaces == null) {
          spaces = 0;
        }
        fr.setLength(fr.getLength() + spaces);
      }  // for each converted fragment
    }  // if spaces should be added (offsets incremented)
    
    return retFragments;
  }  // fragmentsToLinSugFragments()
  
  /**
   * Convert list of common fragments to linearized fragments.
   * For more common fragments one linearized fragment can be created.
   *
   * @param comFragments List of common fragments.
   * @param doc Document in which fragments are
   * @param notConverted List for fragments, which was not converted
   * @param addSpaces If true, it will add space after each node (offset will be incremented by 1)
   * @return Returns list of linearized fragments
   */
  public static ArrayList<Fragment> fragmentsToLinFragments(ArrayList<ArrayList<Fragment>> comFragments,
                                                            Document doc, ArrayList<ArrayList<Fragment>> notConverted,
                                                            boolean addSpaces) {

    // create copy of array from which fragments will be removed and count fragments
    int fragCount = 0;
    int numOfAnnots = comFragments.size();

    ArrayList<ArrayList<Fragment>> comFrCopy = new ArrayList<ArrayList<Fragment>>();
    // array of linearized fragments that will be returned by this method
    ArrayList<Fragment> retFragments = new ArrayList<Fragment>(numOfAnnots);
    // auxiliary array containing fragments from previous document node
    ArrayList<Fragment> partialFragments = new ArrayList<Fragment>(numOfAnnots);
    // auxiliary array containing number of spaces to add before start offset
    ArrayList<Integer> numsOfSpacesSO = new ArrayList<Integer>(numOfAnnots);
    // auxiliary array containing number of spaces to add into length
    ArrayList<Integer> numsOfSpacesL = new ArrayList<Integer>(numOfAnnots);

    for (int i = 0; i < numOfAnnots; i++) {
      ArrayList<Fragment> cFr = comFragments.get(i);
      ArrayList<Fragment> aFragments = new ArrayList<Fragment>();
      notConverted.add(new ArrayList<Fragment>());
      comFrCopy.add(aFragments);
      for (Iterator<Fragment> aFrIt = cFr.iterator(); aFrIt.hasNext();) {
        Fragment fr = aFrIt.next();
        aFragments.add(fr);
        fragCount++;
      }
      // these two arrays should be initialized here, because their size is
      // equal to the number of annotations
      retFragments.add(null);
      partialFragments.add(null);
      numsOfSpacesSO.add(null);
      numsOfSpacesL.add(null);
    }

    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      Node currentNode = nIter.nextNode();
      int nodeCounter = 0;
      String path = "";
      while (currentNode != null && fragCount > 0) {
        // while there is next node and there is something to convert
        nContent = currentNode.getNodeValue();
        int nContentL = nContent.length();
        nEndOffset = nStartOffset + nContentL;
        path = XPathHelper.XPathStringOfNode(currentNode);
        
        for (int i = 0; i < numOfAnnots; i++) {
          // for all sets of fragments
          ArrayList<Fragment> aFragments = comFrCopy.get(i);
          for (Iterator<Fragment> frIt = aFragments.iterator(); frIt.hasNext();) {
            // for all fragments for one annotation
            Fragment fr = frIt.next();
            if (fr.getIsGood() == false) {
              notConverted.get(i).add(fr);  // fragment can not be converted
              frIt.remove();  // fragment was processed
              fragCount--;
              continue;
            }
            if (fr.getPath().equals(path)) {  // if fragment is in the node
              Fragment partFrag = partialFragments.get(i);
              int frEndOffset;
              if (fr.getOffset() != null) {
                frEndOffset = fr.getOffset() + fr.getLength();
              }
              else {
                frEndOffset = fr.getLength();
              }
              if (frEndOffset > nContentL) {  // content of fragment is too long
                notConverted.get(i).add(fr);
                frIt.remove();
                fragCount--;
                continue;
              }   else if (fr.getAnnotatedText() != null && !nContent.substring(fr.getOffset(), frEndOffset).equals(fr.getAnnotatedText())) {
                // fr.getAnnotatedText() returns null when linearizing suggestion fragments
                // content is not matching
                notConverted.get(i).add(fr);
                frIt.remove();
                fragCount--;
                continue;
              }
              if (partFrag == null && frEndOffset < nContentL) {
                // whole linearized fragment in node
                if (fr.getOffset() == null) {
                  // When user requests fragment from the start of a node editor sends offset as null, which causes NullPointerException
                  fr.setOffset(0);
                }
                numsOfSpacesSO.set(i,nodeCounter);  // set number of spaces before start offset for this fragment
                numsOfSpacesL.set(i,0);  // set number of spaces inside length for this fragment
                Fragment linFragment = new Fragment("", nStartOffset + fr.getOffset(),
                                                    fr.getLength(), fr.getAnnotatedText(),
                                                    fr.getRefAnnotation());
                frIt.remove();  // fragment was successfully processed
                fragCount--;
                retFragments.set(i, linFragment);  // set result
              } else if (partFrag == null && frEndOffset == nContentL) {

                // start of linearized fragment in node
                if (fr.getOffset() == null) {
                  fr.setOffset(0);
                }
                numsOfSpacesSO.set(i,nodeCounter);  // set number of spaces before start offset for this fragment
                numsOfSpacesL.set(i,0);  // set number of spaces inside length for this fragment
                Fragment linFragment = new Fragment("", nStartOffset + fr.getOffset(),
                                                    fr.getLength(), fr.getAnnotatedText(),
                                                    fr.getRefAnnotation());

                partialFragments.set(i, linFragment);
                frIt.remove();  // fragment was successfully processed
                fragCount--;
              } else if (partFrag != null && frEndOffset < nContentL) {
                // end of linearized fragment in node
                Fragment linFragment = partialFragments.get(i);
                if (fr.getAnnotatedText() != null) {
                  linFragment.setAnnotatedText(linFragment.getAnnotatedText() + fr.getAnnotatedText());
                }
                linFragment.setLength(linFragment.getLength() + fr.getLength());
                retFragments.set(i, linFragment);
                partialFragments.set(i, null);
                if (numsOfSpacesL.get(i) != null) {  // add space after previous node
                  numsOfSpacesL.set(i,numsOfSpacesL.get(i) + 1);
                }
                frIt.remove();
                fragCount--;
              } else if (partFrag != null && frEndOffset == nContentL) {
                // part of linearized fragment in node
                Fragment linFragment = partialFragments.get(i);
                if (fr.getAnnotatedText() != null) {
                  linFragment.setAnnotatedText(linFragment.getAnnotatedText() + fr.getAnnotatedText());
                }
                linFragment.setLength(linFragment.getLength() + fr.getLength());
                if (numsOfSpacesL.get(i) != null) {  // add space after previous node
                  numsOfSpacesL.set(i,numsOfSpacesL.get(i) + 1);
                }
                frIt.remove();
                fragCount--;
              } else {
                // error state
                notConverted.get(i).add(fr);  // fragment can not be converted
                frIt.remove();  // fragment was processed
                fragCount--;
                continue;
              }
            }  // if fragment is in the node
          }  // for all fragments for one annotation
        }  // for all sets of fragments


        nStartOffset += nContent.length();  // start offset of next node
        currentNode = nIter.nextNode();  // move to next node of document
        while (currentNode != null && (currentNode.getNodeValue() == null || currentNode.getNodeValue().replaceAll("[\\s\\u00A0]+$", "").contentEquals(""))) {
          currentNode = nIter.nextNode();  // move to next node of document
        }
        nodeCounter++;
      }  // while there is next node and there is something to convert
    }  // if document element is presented

    for (int i = 0; i < numOfAnnots; i++) {
      // for all sets of fragments
      ArrayList<Fragment> aFragments = comFrCopy.get(i);
      // move rest of fragments
      for (Iterator<Fragment> aFrIt = aFragments.iterator(); aFrIt.hasNext();) {
        Fragment fr = aFrIt.next();
        notConverted.get(i).add(fr);
        aFrIt.remove();
        fragCount--;
      }

      // finish processing of partial fragment
      if (partialFragments.get(i) != null) {
        retFragments.set(i, partialFragments.get(i));  // set result
      }
    }  // for all sets of fragments

    if (addSpaces) {  // if spaces should be added (offsets incremented)
      for (int i = 0; i < numOfAnnots; i++) {  // for each converted fragment
        Fragment fr = retFragments.get(i);
        if (fr == null) {  // fragment was not converted successfully
          continue;
        }
        // increment offset
        Integer spaces = numsOfSpacesSO.get(i);
        if (spaces == null) {
          spaces = 0;
        }
        fr.setOffset(fr.getOffset() + spaces);
        // increment length
        spaces = numsOfSpacesL.get(i);
        if (spaces == null) {
          spaces = 0;
        }
        fr.setLength(fr.getLength() + spaces);
      }  // for each converted fragment
    }  // if spaces should be added (offsets incremented)
    
    return retFragments;
  }  // fragmentsToLinFragments()

  /**
   * Convert list of common fragments to linearized fragments.
   * For more common fragments one linearized fragment can be created.
   *
   * @param comFragments List of common fragments.
   * @param doc Document in which fragments are
   * @param notConverted List for fragments, which was not converted
   * @param addSpaces If true, it will add space after each node (offset will be incremented by 1)
   * @return Returns list of linearized fragments
   */
  public static ArrayList<AlternativeFragment> fragmentsToLinAltFragments(ArrayList<ArrayList<AlternativeFragment>> comFragments,
                                                            Document doc, ArrayList<ArrayList<AlternativeFragment>> notConverted,
                                                            boolean addSpaces) {

    // create copy of array from which fragments will be removed and count fragments
    int fragCount = 0;
    int numOfAnnots = comFragments.size();

    ArrayList<ArrayList<AlternativeFragment>> comFrCopy = new ArrayList<ArrayList<AlternativeFragment>>();
    // array of linearized fragments that will be returned by this method
    ArrayList<AlternativeFragment> retFragments = new ArrayList<AlternativeFragment>(numOfAnnots);
    // auxiliary array containing fragments from previous document node
    ArrayList<AlternativeFragment> partialFragments = new ArrayList<AlternativeFragment>(numOfAnnots);
    // auxiliary array containing number of spaces to add before start offset
    ArrayList<Integer> numsOfSpacesSO = new ArrayList<Integer>(numOfAnnots);
    // auxiliary array containing number of spaces to add into length
    ArrayList<Integer> numsOfSpacesL = new ArrayList<Integer>(numOfAnnots);

    for (int i = 0; i < numOfAnnots; i++) {
      ArrayList<AlternativeFragment> cFr = comFragments.get(i);
      ArrayList<AlternativeFragment> aFragments = new ArrayList<AlternativeFragment>();
      notConverted.add(new ArrayList<AlternativeFragment>());
      comFrCopy.add(aFragments);
      for (Iterator<AlternativeFragment> aFrIt = cFr.iterator(); aFrIt.hasNext();) {
        AlternativeFragment fr = aFrIt.next();
        aFragments.add(fr);
        fragCount++;
      }
      // these two arrays should be initialized here, because their size is
      // equal to the number of annotations
      retFragments.add(null);
      partialFragments.add(null);
      numsOfSpacesSO.add(null);
      numsOfSpacesL.add(null);
    }

    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is presented
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      Node currentNode = nIter.nextNode();
      int nodeCounter = 0;
      String path = "";
      while (currentNode != null && fragCount > 0) {
        // while there is next node and there is something to convert
        nContent = currentNode.getNodeValue();
        int nContentL = nContent.length();
        nEndOffset = nStartOffset + nContentL;
        path = XPathHelper.XPathStringOfNode(currentNode);
        
        for (int i = 0; i < numOfAnnots; i++) {
          // for all sets of fragments
          ArrayList<AlternativeFragment> aFragments = comFrCopy.get(i);
          for (Iterator<AlternativeFragment> frIt = aFragments.iterator(); frIt.hasNext();) {
            // for all fragments for one annotation
            AlternativeFragment fr = frIt.next();
            if (fr.getIsGood() == false) {
              notConverted.get(i).add(fr);  // fragment can not be converted
              frIt.remove();  // fragment was processed
              fragCount--;
              continue;
            }
            if (fr.getPath().equals(path)) {  // if fragment is in the node
              AlternativeFragment partFrag = partialFragments.get(i);
              int frEndOffset;
              if (fr.getOffset() != null) {
                frEndOffset = fr.getOffset() + fr.getLength();
              }
              else {
                frEndOffset = fr.getLength();
              }
              if (frEndOffset > nContentL) {  // content of fragment is too long
                notConverted.get(i).add(fr);
                frIt.remove();
                fragCount--;
                continue;
              }   else if (fr.getAnnotatedText() != null && !nContent.substring(fr.getOffset(), frEndOffset).equals(fr.getAnnotatedText())) {
                // fr.getAnnotatedText() returns null when linearizing suggestion fragments
                // content is not matching
                notConverted.get(i).add(fr);
                frIt.remove();
                fragCount--;
                continue;
              }
              if (partFrag == null && frEndOffset < nContentL) {
                // whole linearized fragment in node
                if (fr.getOffset() == null) {
                  // When user requests fragment from the start of a node editor sends offset as null, which causes NullPointerException
                  fr.setOffset(0);
                }
                numsOfSpacesSO.set(i,nodeCounter);  // set number of spaces before start offset for this fragment
                numsOfSpacesL.set(i,0);  // set number of spaces inside length for this fragment
                AlternativeFragment linFragment = new AlternativeFragment("", nStartOffset + fr.getOffset(),
                                                    fr.getLength(), fr.getAnnotatedText(),
                                                    fr.getRefAlternative());
                frIt.remove();  // fragment was successfully processed
                fragCount--;
                retFragments.set(i, linFragment);  // set result
              } else if (partFrag == null && frEndOffset == nContentL) {

                // start of linearized fragment in node
                if (fr.getOffset() == null) {
                  fr.setOffset(0);
                }
                numsOfSpacesSO.set(i,nodeCounter);  // set number of spaces before start offset for this fragment
                numsOfSpacesL.set(i,0);  // set number of spaces inside length for this fragment
                AlternativeFragment linFragment = new AlternativeFragment("", nStartOffset + fr.getOffset(),
                                                    fr.getLength(), fr.getAnnotatedText(),
                                                    fr.getRefAlternative());

                partialFragments.set(i, linFragment);
                frIt.remove();  // fragment was successfully processed
                fragCount--;
              } else if (partFrag != null && frEndOffset < nContentL) {
                // end of linearized fragment in node
                AlternativeFragment linFragment = partialFragments.get(i);
                if (fr.getAnnotatedText() != null) {
                  linFragment.setAnnotatedText(linFragment.getAnnotatedText() + fr.getAnnotatedText());
                }
                linFragment.setLength(linFragment.getLength() + fr.getLength());
                retFragments.set(i, linFragment);
                partialFragments.set(i, null);
                if (numsOfSpacesL.get(i) != null) {  // add space after previous node
                  numsOfSpacesL.set(i,numsOfSpacesL.get(i) + 1);
                }
                frIt.remove();
                fragCount--;
              } else if (partFrag != null && frEndOffset == nContentL) {
                // part of linearized fragment in node
                AlternativeFragment linFragment = partialFragments.get(i);
                if (fr.getAnnotatedText() != null) {
                  linFragment.setAnnotatedText(linFragment.getAnnotatedText() + fr.getAnnotatedText());
                }
                linFragment.setLength(linFragment.getLength() + fr.getLength());
                if (numsOfSpacesL.get(i) != null) {  // add space after previous node
                  numsOfSpacesL.set(i,numsOfSpacesL.get(i) + 1);
                }
                frIt.remove();
                fragCount--;
              } else {
                // error state
                notConverted.get(i).add(fr);  // fragment can not be converted
                frIt.remove();  // fragment was processed
                fragCount--;
                continue;
              }
            }  // if fragment is in the node
          }  // for all fragments for one annotation
        }  // for all sets of fragments


        nStartOffset += nContent.length();  // start offset of next node
        currentNode = nIter.nextNode();  // move to next node of document
        while (currentNode != null && (currentNode.getNodeValue() == null || currentNode.getNodeValue().replaceAll("[\\s\\u00A0]+$", "").contentEquals(""))) {
          currentNode = nIter.nextNode();  // move to next node of document
        }
        nodeCounter++;
      }  // while there is next node and there is something to convert
    }  // if document element is presented

    for (int i = 0; i < numOfAnnots; i++) {
      // for all sets of fragments
      ArrayList<AlternativeFragment> aFragments = comFrCopy.get(i);
      // move rest of fragments
      for (Iterator<AlternativeFragment> aFrIt = aFragments.iterator(); aFrIt.hasNext();) {
        AlternativeFragment fr = aFrIt.next();
        notConverted.get(i).add(fr);
        aFrIt.remove();
        fragCount--;
      }

      // finish processing of partial fragment
      if (partialFragments.get(i) != null) {
        retFragments.set(i, partialFragments.get(i));  // set result
      }
    }  // for all sets of fragments

    if (addSpaces) {  // if spaces should be added (offsets incremented)
      for (int i = 0; i < numOfAnnots; i++) {  // for each converted fragment
        AlternativeFragment fr = retFragments.get(i);
        if (fr == null) {  // fragment was not converted successfully
          continue;
        }
        // increment offset
        Integer spaces = numsOfSpacesSO.get(i);
        if (spaces == null) {
          spaces = 0;
        }
        fr.setOffset(fr.getOffset() + spaces);
        // increment length
        spaces = numsOfSpacesL.get(i);
        if (spaces == null) {
          spaces = 0;
        }
        fr.setLength(fr.getLength() + spaces);
      }  // for each converted fragment
    }  // if spaces should be added (offsets incremented)
    
    return retFragments;
  }  // fragmentsToLinAltFragments()


  /**
   * Linearizes fragment of document to string
   *
   * @param docFr Fragment of document to linearize
   * @param doc Whole document with nodes
   * @return Returns linearized fragment of document in string
   */
  public static String linearizeDocumentFragment(DocumentFragment docFr, Document doc) {
    StringBuilder linFr = new StringBuilder();
    NodeList nodeL = docFr.getChildNodes();
    int nodeCount = nodeL.getLength();
    for (int i = 0; i < nodeCount; i++) {  // for each node
      Node curRootNode = nodeL.item(i);
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(curRootNode,
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Node currentNode = nIter.nextNode();
      boolean compensateNewline = false;
      while (currentNode != null) {
        short nodeType = currentNode.getNodeType();
        if (currentNode.getNodeValue() == null) {
          currentNode = nIter.nextNode();  // move to next node of document
          continue;
        }
        if (compensateNewline) {
          linFr.append(" ");
        }
        linFr.append(currentNode.getNodeValue());
        if (!compensateNewline) {
          compensateNewline = true;
        }
        currentNode = nIter.nextNode();  // move to next node of document
      }
    }  // for each node
    return linFr.toString();
  }  // linearizeDocumentFragment()

  /**
   * Convert list of text modifications to linearized modifications.
   * For more common modifications one linearized modification can be created.
   *
   * @param comModifications List of common modifications.
   * @param doc Document in which modifications are
   * @param notConverted Modifications, which was not converted
   * @return Returns list of linearized modifications
   */
  public static ArrayList<TextModification> modificationsToLinMod(ArrayList<TextModification> comModifications,
                                                        Document doc, ArrayList<TextModification> notConverted)
                                                        throws ParserConfigurationException, SAXException,
                                                               IOException {
    if (matcherProvider == null) {  // if matcher provider not exists yet, create it
      matcherProvider = new MatcherProvider();
    }

    // create copy of array from which modifications will be removed and count modifications
    int numOfMods = comModifications.size();
    ArrayList<TextModification> comModCopy = new ArrayList<TextModification>(numOfMods);
    ArrayList<TextModification> notConvertedFlag = new ArrayList<TextModification>(numOfMods);
    notConverted = new ArrayList<TextModification>();

    for (int i = 0; i < numOfMods; i++) {
      TextModification cM = comModifications.get(i);
      comModCopy.add(cM);
      notConvertedFlag.add(cM);
    }

    ArrayList<TextModification> retModifications = new ArrayList<TextModification>();

    if (doc == null) {
      return null;
    }
    if (doc.getDocumentElement() == null) {
      return null;
    } else {  // if document element is present
      NodeIterator nIter = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
      Integer nStartOffset = 0;  // start offset of current node
      Integer nEndOffset = 0;  // end offset of current node
      String nContent = "";  // content of current node
      Node currentNode = nIter.nextNode();

      // ============== adding text to the empty document ============== 
      if (currentNode == null) {
        int nodeCounter = 0;
        int modifStart = 0;
        for (Iterator<TextModification> tmIt = comModCopy.iterator(); tmIt.hasNext();) {
          // for all modifications
          TextModification tm = tmIt.next();

          // replace the line break tag with a space
          tm.setNewContent(tm.getNewContent().replace("<br>"," "));
          // remove all common HTML tags so the content is linearized
          tm.setNewContent(tm.getNewContent().replaceAll("<[^>\\s]*>",""));
          tm.setLength(tm.getNewContent().length());

          TextModification lTM = new TextModification(null, modifStart,
                                                            modifStart + tm.getLength(), tm.getNewContent());
          modifStart += tm.getLength();
          retModifications.add(lTM);
          notConvertedFlag.set(nodeCounter, null);  // modification successfully converted
          numOfMods--;
          nodeCounter += 1;
        }
      }

      // ============== modification of the non-empty document ============== 
      else {
        String path = "";
        int modifCounter; // index to the wasConverted flag array
        boolean compensateNewline = false;
        while (currentNode != null && numOfMods > 0) {
          // while there is next node and there is something to convert
          if ( compensateNewline ) {
            // add a space character to prevent word joining at the end and at the beginning
            nContent = currentNode.getNodeValue().concat(" ");
          }
          else {
            nContent = currentNode.getNodeValue();
          }
          int nContentL = nContent.length();
          nEndOffset = nStartOffset + nContentL;
          path = XPathHelper.XPathStringOfNode(currentNode);

          modifCounter = 0;
          for (Iterator<TextModification> tmIt = comModCopy.iterator(); tmIt.hasNext();) {
            // for all modifications
            TextModification tm = tmIt.next();

            if (tm.getPath() == null || tm.getPath().isEmpty()) {
                // linearized modification - no linearization needed
                retModifications.add(tm);
                notConvertedFlag.set(modifCounter, null);  // modification successfully converted
                numOfMods--;
                continue;
            }
            
            if (tm.getOffset() == null && tm.getLength() == null && tm.getPath().equals("/HTML[1]/BODY[1]")) {
              // modification of the whole body
              // create iterator and initialize auxiliary variables
              NodeIterator wholeDocEraseIt = ((DocumentTraversal) doc).createNodeIterator(doc.getDocumentElement(),
                            NodeFilter.SHOW_TEXT + NodeFilter.SHOW_CDATA_SECTION, null, true);
              Node n = wholeDocEraseIt.nextNode();
              boolean newlineCompens = false;
              int docLength = 0;
              String nodeCont = null;

              // determine the length of the old version of the document
              while (n != null) {
                nodeCont = n.getNodeValue();
                if (nodeCont == null) {
                  n = wholeDocEraseIt.nextNode();
                  continue;
                }
                docLength += nodeCont.length();
                if (newlineCompens) {
                  docLength += 1;
                }
                if (!newlineCompens) {
                  newlineCompens = true;
                }
                n = wholeDocEraseIt.nextNode();
              }
                
              String newContent = "";
              // if the new content doesn't represent empty text data set,
              // remove all HTML tags
              if (tm.getNewContent() != null && !tm.getNewContent().equals("<body></body>")) {
                newContent = tm.getNewContent();
                // replace the line break tag with a space
                newContent = newContent.replace("<br>"," ");
                // remove all common HTML tags so the content is linearized
                newContent = newContent.replaceAll("<[^>\\s]*>","");
              }

              // assemble the linearized version of the modification
              TextModification lTM = new TextModification(null, 0, docLength, newContent);
              retModifications.add(lTM);
              notConvertedFlag.set(modifCounter, null);  // modification successfully converted
              numOfMods--;
              continue;
            }

            if (path.startsWith(tm.getPath())) {  // if modification is in the node
              if (tm.getOffset() == null) {
                // if whole content of the node will be replaced
                String linContent = "";
                if (tm.getNewContent() != null) { // whole node content has been removed
                  linContent = linearizeDocumentFragment(matcherProvider.getFragmentFromString(tm.getNewContent()), doc);
                }

                // perform the linearization process and assemble the linearized
                // modification data structure
                TextModification lTM = null;
                if (tm.getOffset() == null) {
                  lTM = new TextModification(null, nStartOffset,
                                                            nEndOffset - nStartOffset, linContent);
                }
                else {
                  lTM = new TextModification(null, nStartOffset + tm.getOffset(),
                                                            nEndOffset - nStartOffset, linContent);
                }

                retModifications.add(lTM);
                notConvertedFlag.set(modifCounter, null);  // modification successfully converted
                numOfMods--;
                continue;
              }
              
              // ------ modified fragment is too long -------
              if (tm.getOffset() != null && tm.getLength() != null) {
                int tmEndOffset = tm.getOffset() + tm.getLength();

                if (tmEndOffset > nContentL) {  // modified fragment is too long
                  notConverted.add(tm);
                  notConvertedFlag.set(modifCounter, null);;
                  numOfMods--;

                  continue;
                }
              }
            } // modification is in the node
          }  // for all modifications
          nStartOffset += nContent.length() + 1;  // start offset of next node
                                                  // +1 for newline compensation
          currentNode = nIter.nextNode();  // move to next node of document
        }  // while there is next node and there is something to convert
      } // modification of the non-empty document
    }  // if document element is presented

    // move rest of the modifications to the not converted list
    for (Iterator<TextModification> tmIt = notConvertedFlag.iterator(); tmIt.hasNext();) {
      TextModification tm = tmIt.next();
      if (tm != null) {
        notConverted.add(tm);
      }
    }

    return retModifications;
  }  // modificationsToLinMod()

}  // public class Linearizer