DocumentClone.java

Go to the documentation of this file.

/*
 * Project: Server for annotations sharing
 * Author: Ing. Jaroslav Dytrych idytrych@fit.vutbr.cz
 * File: DocumentClone.java
 * Description: Class provides document clonning with new content, iniciated from NLP.
 */

/**
 * @file DocumentClone.java
 *
 * @brief Class provides document clonning with new content, iniciated from NLP.
 */

package cz.vutbr.fit.knot.annotations.documentCloner;

import cz.vutbr.fit.knot.annotations.app.AppBean;
import cz.vutbr.fit.knot.annotations.app.Constants;
import cz.vutbr.fit.knot.annotations.comet.Util;
import cz.vutbr.fit.knot.annotations.entity.AnnotDocument;
import cz.vutbr.fit.knot.annotations.entity.Annotation;
import cz.vutbr.fit.knot.annotations.entity.Fragment;
import cz.vutbr.fit.knot.annotations.entity.attribute.BaseAttribute;
import cz.vutbr.fit.knot.annotations.entity.attribute.LinkedAnnotationAttribute;
import cz.vutbr.fit.knot.annotations.entity.attribute.NestedAnnotationAttribute;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.Comparator;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.Matcher;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.MatcherProvider;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.UpdatableFragment;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.compareMethods.ExactMethod;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.compareMethods.LevenshteinMethod;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.compareMethods.MetaphoneMethod;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.nodeIterators.BidirectionallyUnNestNodeIterator;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.nodeIterators.SequenceNodeIterator;
import cz.vutbr.fit.knot.annotations.fragmentUpdater.nodeIterators.XPathNodeIterator;
import cz.vutbr.fit.knot.annotations.modules.StoryscopeInterface.StoryscopeInterfaceSender;
import java.io.BufferedReader;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.servlet.http.HttpServletRequest;
import javax.xml.xpath.XPathExpressionException;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.w3c.dom.Document;

/**
 * Class provides document clonning with new content, iniciated from NLP.
 *
 * @brief Class provides document clonning with new content, iniciated from NLP
 * @author Martin Petr (xpetrm05)
 */
public class DocumentClone {
  /** Name of old document uri parameter in JSON string */
  public static final String OLD_DOCUMENT_URI_PARAM = "oldDocumentUri";
  
  /** Name of new document uri parameter in JSON string */
  public static final String NEW_DOCUMENT_URI_PARAM = "newDocumentUri";
  
  /** Name of new document content parameter in JSON string */
  public static final String NEW_DOCUMENT_CONTENT_PARAM = "content";
  
  /** Provider of matchers for finding fragments (exact matches only) */
  private MatcherProvider exactMatcherProvider;
  
  /** Provider of matchers for finding fragments (nearest matches only) */
  private MatcherProvider nearestMatcherProvider;
  
  /** Provider of matchers for finding fragments (sequence iterator) */
  private MatcherProvider sequenceMatcherProvider;

  /**
   * Constructor
   */
  public DocumentClone() {
    exactMatcherProvider = new MatcherProvider();
    nearestMatcherProvider = new MatcherProvider();
    sequenceMatcherProvider = new MatcherProvider();
    
    // create nearest matcher
    nearestMatcherProvider.add(new Matcher(new BidirectionallyUnNestNodeIterator(), new Comparator(new ExactMethod(), Comparator.TraversingMethod.CHARACTER_BIDIRECTIONALLY)), 0);
    nearestMatcherProvider.add(new Matcher(new BidirectionallyUnNestNodeIterator(), new Comparator(new LevenshteinMethod(0.8), Comparator.TraversingMethod.WORD)), 1);
    nearestMatcherProvider.add(new Matcher(new BidirectionallyUnNestNodeIterator(), new Comparator(new LevenshteinMethod(0.8), Comparator.TraversingMethod.FIRST_AND_LAST_WORD_THEN_LETTER, 2.0, 2.0)), 2);
    
    // create sequence mathcer
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new ExactMethod(), Comparator.TraversingMethod.CHARACTER_BIDIRECTIONALLY)), 0);
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new LevenshteinMethod(0.8), Comparator.TraversingMethod.FIRST_AND_LAST_WORD_THEN_LETTER, 2.0, 2.0)), 1);
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new LevenshteinMethod(0.5), Comparator.TraversingMethod.WORD)), 2);
    sequenceMatcherProvider.add(new Matcher(new SequenceNodeIterator(), new Comparator(new MetaphoneMethod(), Comparator.TraversingMethod.CHARACTER_BIDIRECTIONALLY)), 3);

    // create exact matcher
    exactMatcherProvider.add(new Matcher(new XPathNodeIterator(), new Comparator(new ExactMethod(), Comparator.TraversingMethod.STATIC)), 0);
  }

  /**
   * Method provides clone of old document to new document with new content. Method 
   * check if server is approved to send request for clone. Method also clone 
   * annotations of old document.
   * 
   * @param request request from servlet
   * @return message for server
   */
  public String cloneDocument(HttpServletRequest request){
    CloneParams actualParams = new CloneParams(request);
    if(isAuthorized(actualParams)){
      //request from authorized server
      try{
        parseJSON(actualParams);
      }catch(Exception e){
        //error in parsed JSON string
        if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_ALL) {
          String msg = "Parse JSON in document clone error: " + e.getMessage();
          Logger.getLogger(DocumentClone.class.getName()).log(Level.ALL, msg);
        }
      }
    }else{
      //error unauthorized request
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_ALL) {
        String msg = "Unauthorized document clone request from IP: " + request.getRemoteAddr();
        Logger.getLogger(DocumentClone.class.getName()).log(Level.ALL, msg);
      }
      return "";
    }
    
    // here is authorized request and parsed paramethers of request
    List<Annotation> oldAnnotList = null;
    try{
      //make object with parsed new document
      parseDocument(actualParams);
      //make and store new document to DB
      createNewDocument(actualParams);
      
      oldAnnotList = getAllAnnotations(actualParams);
      if(oldAnnotList != null && !oldAnnotList.isEmpty()){
        //there are some annotations in document
        makeNewAnnotations(oldAnnotList,actualParams);
      }else{
        //no annotations in ndocument clone is comeplete
        return "ok";
      }
    }catch(Exception e){
      if (Constants.LOG_LEVEL >= Constants.LOG_LEVEL_ALL) {
        String msg = "Error while creating new document: " + e.getMessage();
        System.out.println(msg);
        Logger.getLogger(DocumentClone.class.getName()).log(Level.ALL, msg);
      }
    }
    
    // inform StoryScope - tripleStore 
    if(oldAnnotList != null && !oldAnnotList.isEmpty()){
      StoryscopeInterfaceSender sender = new StoryscopeInterfaceSender("StoryscopeInterfaceSender for DocumentClonner");
      ArrayList<Annotation> tmpArray = new ArrayList<Annotation>(oldAnnotList.size());
      tmpArray.addAll(oldAnnotList);
      sender.setAddedAnnotations(tmpArray);
      sender.start();
    }
    return "";
  }
  
  /** 
   * Method checks if request come from authorized server.
   * 
   * @param params object with parameters of request
   * @return true value indicate thet server is authorized, false indicates unauthorized request
   */
  private boolean isAuthorized(CloneParams params){
    @SuppressWarnings("unchecked")
    List<DocClonerServer> results = AppBean.getPersistenceManager().getEntitiesByName("DocClonerServer");
    if(results == null || results.isEmpty()){
      return true;
    }
    
    String address = params.getRequest().getRemoteAddr();
    Iterator<DocClonerServer> resultsIt = results.iterator();
    if(resultsIt.hasNext()){
      if(resultsIt.next().getIpAddress().equals(address)){
        return true;
      }
    }

    return false;
  }
  
  
  /**
   * Method parse JSON string to parameters and save them to params object. If 
   * problem occurs during parsing, method throws exception with message.
   * 
   * @param params object with parameters of request
   * @return method returns true if parsing ends with succes otherwise returns false
   * @throws RuntimeException if it fails to find the request parameters
   * 
   * Simple example of JSON string:
   * 
   * {
   *   "oldDocumentUri": "oneValue",
   *   "newDocumentUri": "oneValue",
   *   "content": "abcd content cdef"
   * }
   */
  private boolean parseJSON(CloneParams params)
          throws RuntimeException{
    String requestMessage = readMessage(params);
    if(requestMessage != null && !requestMessage.isEmpty()){ 
      JSONObject jsonResponse = (JSONObject)JSONValue.parse(requestMessage);
      if(jsonResponse != null){
        
        try{
          // get uri of old document
          params.setOldDocumentUri(getJsonParameter(jsonResponse,OLD_DOCUMENT_URI_PARAM));
          // get uri of new document
          params.setNewDocumentUri(getJsonParameter(jsonResponse,NEW_DOCUMENT_URI_PARAM));
          // get conetent of new document
          params.setNewDocumentContent(getJsonParameter(jsonResponse,NEW_DOCUMENT_CONTENT_PARAM));
        }catch(Exception e){
          //re-throw occured exception 
          throw new RuntimeException(e.getMessage());
        }        
      }else{
        //error can't parse recived message
        throw new RuntimeException("Can't parse JSON string.");
      }
    }else{
      //can't read request string
      throw new RuntimeException("Can't read JSON request string.");
    }
    return false;
  }
  
  
  /**
   * Method creates a new document by the source document.
   * 
   * @param params object with parameters of request
   * @return true if creating of new document ends with success
   * @throws RuntimeException if creating new document fails
   */
  private boolean createNewDocument(CloneParams params)
          throws RuntimeException{
    
    AnnotDocument checkDoc = findDocumentInDB(params.getNewDocumentUri());
    if(checkDoc != null) {
        throw new RuntimeException("Document with new document URI: " + params.getNewDocumentUri() + " is already in DB.");
    }
    // at first is important to fetch old document from DB
    AnnotDocument oldDocument = findDocumentInDB(params.getOldDocumentUri());
    params.setOldDocument(oldDocument);
    if(oldDocument != null){
      // make new document 
      AnnotDocument newDocument = new AnnotDocument();
      newDocument.setUri(params.getNewDocumentUri());
      newDocument.setContent(params.getNewDocumentContent());
      newDocument.setModified(new Date());
      newDocument.setAdded(oldDocument.getAdded());
      
      // store new document to DB
      if (AppBean.getPersistenceManager().persistEntity(newDocument)) {
        throw new RuntimeException("Persisting of new document failed.");
      } 
      
      params.setNewDocument(newDocument);
      
    }else{
      //error can't find document in DB
      throw new RuntimeException("Can't find document with uri: " + params.getOldDocumentUri() + " in DB.");
    }

    return false;
  }
  
  /**
   * The method creates a new annotation according to the source document.
   * 
   * @param oldAnnotations List of old annotations to clone
   * @param params object with parameters of request
   * @return true if copying of new annotations ends with success
   * @throws RuntimeException if creating new annotations fails
   */
  private boolean makeNewAnnotations(List<Annotation> oldAnnotations, CloneParams params)
          throws RuntimeException {
    //At first clone annotations
    Iterator<Annotation> oldAnnotIt = oldAnnotations.iterator();
    //clone and create list of clonned annotations
    while(oldAnnotIt.hasNext()){
      Annotation currentAnnot = oldAnnotIt.next();
      AnnotCloneItem newItem = new AnnotCloneItem(currentAnnot.getId(),currentAnnot);
      newItem.setNewAttributesOfAnnotation(getAllAttributes(currentAnnot.getId()));
      
      //change old annotation to new
      currentAnnot.setId(null);
      currentAnnot.setAttributes(null);
      currentAnnot.setSourceDocument(params.getNewDocument());
      currentAnnot.setSourceDocumentId(params.getNewDocument().getId());
      
      Iterator<Fragment> fragmentsIt = currentAnnot.getFragments().iterator();
      while(fragmentsIt.hasNext()){
        fragmentsIt.next().setRefAnnotation(currentAnnot);
      }
      //update annotation
      updateAnnotation(currentAnnot, params);
      
      //persist annotation to DB
      if(AppBean.getPersistenceManager().persistEntity(currentAnnot)){
        //error can't persist parameter of annotation to DB
        throw new RuntimeException("Can't persist annotation to DB.");
      }
      
      //add information about cloned annotation to list
      params.addCloneAnnotation(newItem);
    }
    
    //As second add attributes to clonned annotations
    Iterator<AnnotCloneItem> clonedAnnotIt = params.getCloneAnnotations().iterator();
    while(clonedAnnotIt.hasNext()){
      AnnotCloneItem currentItem = clonedAnnotIt.next();
      //check if old annotation had any attributes

      if(currentItem.getNewAttributesOfAnnotation() != null && !currentItem.getNewAttributesOfAnnotation().isEmpty()){
        Iterator<BaseAttribute> attributeIt = currentItem.getNewAttributesOfAnnotation().iterator();
        //Go trough all attributes
        while(attributeIt.hasNext()){
          BaseAttribute currentAttribute = attributeIt.next();
          //Test if current attribute is annotation link
          if(currentAttribute instanceof LinkedAnnotationAttribute){
            //Linked attribute
            LinkedAnnotationAttribute tmpAttribute = (LinkedAnnotationAttribute)currentAttribute;
            Annotation linkedAnnotation = (Annotation)tmpAttribute.getValue();
            if(linkedAnnotation != null){
              AnnotCloneItem foundedAnnotation = params.findCloneAnnotation(linkedAnnotation.getId());
              tmpAttribute.setValue(foundedAnnotation.getNewAnnotation());
              if(foundedAnnotation.getNewAnnotation() != null){
                tmpAttribute.setAttributeType(foundedAnnotation.getNewAnnotation().getAnnotType());
              }
            }
          }else if(currentAttribute instanceof NestedAnnotationAttribute){
            //Nested attribute
            NestedAnnotationAttribute tmpAttribute = (NestedAnnotationAttribute)currentAttribute;
            Annotation nestedAnnotation = (Annotation)tmpAttribute.getValue();
            if(nestedAnnotation != null){
              AnnotCloneItem foundedAnnotation = params.findCloneAnnotation(nestedAnnotation.getId());
              tmpAttribute.setValue(foundedAnnotation.getNewAnnotation());
              if(foundedAnnotation.getNewAnnotation() != null){
                tmpAttribute.setAttributeType(foundedAnnotation.getNewAnnotation().getAnnotType());
              }
            }
          }
          //Normal attribute 
          currentAttribute.setAnnotation(currentItem.getNewAnnotation().getId());
          //Add attribute into DB
          if(AppBean.getPersistenceManager().persistEntity(currentAttribute)){
            //error can't persist parameter of annotation to DB
            throw new RuntimeException("Can't persist parameter of annotation to DB.");
          }
        }
      }
    }
    
    return true;
  }  // makeNewAnnotations()
  
  /** 
   * Method reads content of POST message.
   * 
   * @param params object with parameters of request
   * @return string with content of message
   */
  private String readMessage(CloneParams params){
    StringWriter writer = new StringWriter();
    try{
      BufferedReader bufReader = params.getRequest().getReader();
      int nextChar;
      while ((nextChar = bufReader.read()) != -1) {
        writer.write(nextChar);
      }
    }catch(Exception e){
      return null;
    }
    return writer.toString();
  }
  
  /** 
   * Method parse specific request parameter in JSON string. If there is more 
   * then one parameter with given name, is returned first found.
   * 
   * @param jsonResponse JSON object with parameters
   * @param parameterName Name of specific parameter
   * @return value of parameter
   * @throws RuntimeException if parsing of JSON parameters fails
   */
  private String getJsonParameter(JSONObject jsonResponse, String parameterName)
          throws RuntimeException{
    String result = null;
    if(jsonResponse.containsKey(parameterName)){
      //get first found
      result = (String)jsonResponse.get(parameterName);
      if(result == null || result.isEmpty()){
        //error old document parameter id empty
        throw new RuntimeException("Parameter " + parameterName + " is empty.");
      }
    }else{
      //error can't find old document uri
      throw new RuntimeException("Can't find " + parameterName + " parameter in JSON string.");
    }
    
    return result;
  }
  
  /**
   * Method parse given document and save it to CloneParams object. Method throws 
   * exception if error occurs. 
   * 
   * @param params object with parameters of request
   */
  private void parseDocument(CloneParams params){
    // method builds old document
    try {
      MatcherProvider mp = new MatcherProvider();
      Document parsedSyncDocument = mp.getDocumentFromString(params.getNewDocumentContent(), false, true);
      params.setParsedDoc(parsedSyncDocument);
    } catch (Exception ex) {
      //error can't build document
      throw new RuntimeException("Can't parse document:" + ex.getMessage() + ".");
    }
  }

  /**
   * Updates fragments in annotation. (Method copied from CoreFuncMOdule)
   *
   * @param annotation Annotation to update
   * @param params Parameters from request
   * @return If annotation was updated, returns true, if no update needed or
   * error occurred, returns false
   */
  private boolean updateAnnotation(Annotation annotation, CloneParams params) {
    if (annotation.getFragments().size() < 1) {
      // if annotation haven't fragments, it can't be updated
      return false;
    }

    // get synchronized document
    AnnotDocument annotDoc = params.getNewDocument();
    Document parsedDoc = params.getParsedDoc();
    if (parsedDoc == null) {
      //make message about error and throw it
      //"Bad document for update of the annotation."
      return false;  // updating is not possible
    }

    int badFragCnt = 0;  // total count of bad fragments
    int nowOrphFragCnt = 0;  // count of just orphaned fragments
    int updatedFragCnt = 0;  // count of just updated fragments
    Iterator<Fragment> fragIt = annotation.getFragments().iterator();
    while (fragIt.hasNext()) {  // for each annotated fragment
      Fragment fragment = fragIt.next();
      try {
        // find fragment
        UpdatableFragment uf = exactMatcherProvider.match(parsedDoc, fragment.toUpdatableFragment());
        if (uf != null) {  // if fragment was found
          if (fragment.getIsGood() == false) {  // if fragment was marked as bad
            fragment.setIsGood(true);  // mark as good
            updatedFragCnt++;  // fragment was now updated
          }
        } else {  // if fragment was not found
          // find fragment in document with full matching capability
          //ArrayList <UpdatableFragment> ufl = matcherProvider.matchAllIncrementally(parsedDoc, fragment.toUpdatableFragment());
          ArrayList<UpdatableFragment> ufl = new ArrayList<UpdatableFragment>();
          // find nearest fragment
          UpdatableFragment ufTemp = nearestMatcherProvider.matchInClosestNode(parsedDoc, fragment.toUpdatableFragment());
          if (ufTemp != null) {
            ufl.add(ufTemp);
          } else {
            // find fragment with sequence iterator
            ufl = sequenceMatcherProvider.matchAllIncrementally(parsedDoc, fragment.toUpdatableFragment());
          }
          if (ufl.size() == 1) {  // if one fragment was found
            // get fragment (index unknown)
            Iterator<UpdatableFragment> ufIt = ufl.iterator();
            uf = ufIt.next();
            fragment.updateWithUpdatableFragment(uf);  // update fragment in annotation
            updatedFragCnt++;
            if (fragment.getIsGood() == false) {  // if fragment was marked as bad
              fragment.setIsGood(true);  // mark as good
            }
          } else if (ufl.isEmpty()) {  // if no fragment found
            if (fragment.getIsGood() == true) {  // if fragment was not bad
              fragment.setIsGood(false);  // mark as bad
              nowOrphFragCnt++;  // fragment was now orphaned
            }
            badFragCnt++;
          } else {  // if searching is ambiguous
            // select the best fragment
            int minBadness = Integer.MAX_VALUE;  // minimal badness
            UpdatableFragment minBFR = null;  // fragment with minimal L.D.
            String origPath = fragment.getPath();  // original XPath
            String origText = fragment.getAnnotatedText();  // original annotated text
            int origOffset = fragment.getOffset();  // original offset
            for (Iterator<UpdatableFragment> uflIt = ufl.iterator(); uflIt.hasNext();) {
              UpdatableFragment ufr = uflIt.next();
              int lD = Util.levenshtein(origPath, ufr.getXPathString());
              int lDT = Util.levenshtein(origText, ufr.getText());
              int offDistance = Math.abs(origOffset - ufr.getOffset());
              int badness = 10000 * lD + 1000 * lDT + offDistance;
              if (badness < minBadness) {
                minBadness = badness;
                minBFR = ufr;
              }
            }
            uf = minBFR;
            fragment.updateWithUpdatableFragment(uf);  // update fragment in annotation
            updatedFragCnt++;
            if (fragment.getIsGood() == false) {  // if fragment was marked as bad
              fragment.setIsGood(true);  // mark as good
            }
          }  // if searching is ambiguous
        }  // if fragment was not found
      } catch (XPathExpressionException ex) {  // bad XPath in fragment
        if (fragment.getIsGood() == true) {  // if fragment was not bad
          fragment.setIsGood(false);  // mark as bad
          nowOrphFragCnt++;  // fragment was now orphaned
        }
        badFragCnt++;
      } catch (Exception e) {
        //make and throw error
        //"Exception during the update of the annotation."
        badFragCnt++;
        break;
      }
    }  // for each annotated fragment

    if (badFragCnt == annotation.getFragments().size() && nowOrphFragCnt > 0) {
      // if annotation was just orphaned
    } else if (nowOrphFragCnt > 0) {
      // if annotation was just partially orphaned (bad fragments found)
    }
    if (nowOrphFragCnt > 0 || updatedFragCnt > 0) {
      // if new bad fragments found or some fragments was updated
      return true;
    }
    return false;  // nothing updated (no update needed)
  }  // updateAnnotation()
  
  /** 
   * Method finds document in database by given uri. If don't find returns null.
   * 
   * @param uri Uri of document
   * @return document that is specified by given uri or null 
   */
  private AnnotDocument findDocumentInDB(String uri){
    Object[] params = new Object[2];
    params[0] = "uri";
    params[1] = uri;
    
    @SuppressWarnings("unchecked")
    List<AnnotDocument> results = (List<AnnotDocument>) AppBean.getPersistenceManager().queryDB("AnnotDocument.findByUri", params);
    if(results != null && !results.isEmpty()){
      return results.get(0);
    }
    
    return null;
  }
  
  /** 
   * Method finds all annotation belongs to old document. If don't find returns null.
   * 
   * @param params object with parameters of request
   * @return list of annotations that belongs to old document
   */
  private List<Annotation> getAllAnnotations(CloneParams params){
    Object[] paramsDB = new Object[2];
    paramsDB[0] = "sourceDocumentId";
    paramsDB[1] = params.getOldDocument().getId();
    
    @SuppressWarnings("unchecked")
    List<Annotation> retList = AppBean.getPersistenceManager().queryDB("Annotation.findBySourceDocumentID", paramsDB);
    
    return retList;
  }
  
  /** 
   * Method finds all attributes of annotation. If don't find returns null.
   * 
   * @param id Id of annotation
   * @return Returns list of attributes that belongs to given annotation
   */
  private List<BaseAttribute> getAllAttributes(Integer id){
    Object[] paramsDB = new Object[2];
    paramsDB[0] = "annotation";
    paramsDB[1] = id;
    
    @SuppressWarnings("unchecked")
    List<BaseAttribute> retList = AppBean.getPersistenceManager().queryDB("Attribute.findByAnnotation", paramsDB);
    
    return retList;
  }
}  // public class DocumentClone