Code, binary, data, and README

README.txt

@@ -0,0 +1,60 @@
+About
+========
+
+GraphSeg is a tool for semantic/topical segmentation of text that employs semantic relatedness and a graph-based algorithm to identify semantically coherent segments in text. 
+Segmentation is performed at the sentence level (no intra-sentential segment beginnings/end)
+
+Content
+========
+
+This repository contains: 
+
+(1) the Java source code (as Maven project)
+(2) the ready-to-use binary version of the tool (graphseg.jar in the /binary folder) 
+(3) the dataset of political manifestos manually annotated with segments (used for evaluation in the research paper the GraphSeg tool accompanies). 
+
+Usage 
+========
+
+The following command with four arguments runs the GraphSeg tool:
+
+java -jar graphseg.jar <input-folder-path> <output-folder-path> <relatedness-treshold> <minimal-segment-size>
+
+The argument (all mandatory) to be provided are:
+
+(1) <input-folder-path> is the path to the folder (directory) containing the raw text documents that need to be topically/semantically segmented;
+(2) <output-folder-path> is the path to the folder in which the semantically/topically segmented input documents are to be stored;
+(3) <relatedness-treshold> is the value of the relatedness treshold (decimal number) to be used in the construction of the relatedness graph: larger values will give large number of smalled segments, whereas the smaller treshold values will provide a smaller number of coarse segments;
+(4) <minimal-segment-size> defines the minimal segment size m (in number of sentences). This means that GraphSeg will not produce segments containing less than m sentences.    
+     
+Example command: 
+
+java -jar graphseg.jar /home/seg-input /home/seg-output 0.25 3
+
+Credit
+========
+
+In case you use GraphSeg in your research, please give approproate credit to our work by citing the following publication: 
+
+@InProceedings{glavavs-nanni-ponzetto:2016:*SEM,
+  author    = {Glava\v{s}, Goran  and  Nanni, Federico  and  Ponzetto, Simone Paolo},
+  title     = {Unsupervised Text Segmentation Using Semantic Relatedness Graphs},
+  booktitle = {Proceedings of the Fifth Joint Conference on Lexical and Computational Semantics},
+  month     = {August},
+  year      = {2016},
+  address   = {Berlin, Germany},
+  publisher = {Association for Computational Linguistics},
+  pages     = {125--130},
+  url       = {http://anthology.aclweb.org/S16-2016}
+}
+
+Contact
+========
+
+Please address all questions about the GraphSeg tool and the *SEM publication to: 
+
+Dr. Goran Glava�
+Data and Web Science Group
+University of Mannheim
+
+Email: [email protected]

binary/graphseg.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83ac4ce85663bd97072a2fad76349bf923d1869b7acd7a67f797e6c16a1a47b2
+size 350762888

source/pom.xml

@@ -0,0 +1,85 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>edu.uma.nlp.graphseg</groupId>
+  <artifactId>graphseg</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
+  <name>graphseg</name>
+  <description>Textual segmentation using graph-based algorithm using semantic relatedness</description>
+  <build>
+		<sourceDirectory>src</sourceDirectory>
+		<resources>
+			<resource>
+				<directory>src</directory>
+				<excludes>
+					<exclude>**/*.java</exclude>
+				</excludes>
+			</resource>
+		</resources>
+		<plugins>
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.3</version>
+				<configuration>
+					<source>1.8</source>
+					<target>1.8</target>
+				</configuration>
+			</plugin>
+			<plugin>
+      			<artifactId>maven-assembly-plugin</artifactId>
+      			<configuration>
+        			<archive>
+ 			         	<manifest>
+            				<mainClass>edu.uma.nlp.graphseg.Start</mainClass>
+        			 	</manifest>
+        			</archive>
+        			<descriptorRefs>
+          				<descriptorRef>jar-with-dependencies</descriptorRef>
+        			</descriptorRefs>
+      			</configuration>
+      			<executions>
+				  <execution>
+					<id>make-assembly</id>
+                                        <!-- bind to the packaging phase -->
+					<phase>package</phase> 
+					<goals>
+						<goal>single</goal>
+					</goals>
+				  </execution>
+				</executions>
+    		</plugin>
+		</plugins>
+	</build>
+  <dependencies>
+	<dependency>
+	  <groupId>org.jgrapht</groupId>
+ 		  <artifactId>jgrapht-core</artifactId>
+ 		  <version>0.9.1</version>
+	</dependency>
+	<dependency>
+		<groupId>org.javatuples</groupId>
+		<artifactId>javatuples</artifactId>
+		<version>1.2</version>
+	</dependency>
+	<dependency>
+		<groupId>commons-io</groupId>
+		<artifactId>commons-io</artifactId>
+		<version>2.4</version>
+	</dependency>
+	<dependency>
+		<groupId>org.apache.commons</groupId>
+		<artifactId>commons-lang3</artifactId>
+		<version>3.4</version>
+	</dependency>
+	<dependency>
+		<groupId>edu.stanford.nlp</groupId>
+		<artifactId>stanford-corenlp</artifactId>
+		<version>3.5.2</version>
+	</dependency>
+	<dependency>
+		<groupId>edu.stanford.nlp</groupId>
+		<artifactId>stanford-corenlp</artifactId>
+		<version>3.5.2</version>
+		<classifier>models</classifier>
+	</dependency>	
+  </dependencies>
+</project>

source/src/config.properties

@@ -0,0 +1,3 @@
+inf-cont-path=C:/Goran/Corpora/unigram-freqs-english.txt
+word-vec-path=C:/Goran/Corpora/WordVectors/glove-vectors-6b-200d.txt
+stop-words-path=C:/Goran/Corpora/stopwords.txt

source/src/edu/uma/nlp/graphseg/ClusteringHandler.java

@@ -0,0 +1,206 @@
+package edu.uma.nlp.graphseg;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class ClusteringHandler {
+	
+	public List<List<Integer>> getSequentialClusters(List<List<Integer>> cliques, Map<Integer, Map<Integer, Double>> allSimilarities, int largestTooSmallClusterSize)
+	{
+		List<List<Integer>> sequentialClusters = new ArrayList<List<Integer>>();
+		
+		System.out.println("Merging cliques...");
+		mergeCliques(cliques, sequentialClusters);
+		System.out.println("Merging singletons...");
+		mergeSingletons(cliques, sequentialClusters, allSimilarities);
+		System.out.println("Merging too small sequences...");
+		mergeTooSmallSequences(sequentialClusters, allSimilarities, largestTooSmallClusterSize);
+		
+		return sequentialClusters;
+	}
+	
+	private void mergeCliques(List<List<Integer>> cliques, List<List<Integer>> sequentialClusters)
+	{
+		boolean change = true;
+		while(change)
+		{
+			change = false;
+			for(List<Integer> clique : cliques)
+			{
+				for(int i = 0; i < clique.size() - 1; i++)
+				{
+					for(int j = i+1; j < clique.size(); j++)
+					{
+						int ind = i;
+						int jond = j;
+						Optional<List<Integer>> existingClusterFirst = sequentialClusters.stream().filter(sc -> sc.contains(clique.get(ind))).findFirst();
+						Optional<List<Integer>> existingClusterSecond = sequentialClusters.stream().filter(sc -> sc.contains(clique.get(jond))).findFirst();
+						
+						// Both nodes from the clique already placed in clusters
+						if (existingClusterFirst.isPresent() && existingClusterSecond.isPresent())
+						{
+							continue;
+						}
+						
+						// Neither of the nodes is in the cluster
+						else if (!existingClusterFirst.isPresent() && !existingClusterSecond.isPresent())
+						{
+							// if these are consecutive sentences, we make a new cluster
+							if (Math.abs(clique.get(i) - clique.get(j)) == 1)
+							{
+								List<Integer> newCluster = new ArrayList<Integer>();
+								newCluster.add(Math.min(clique.get(i), clique.get(j)));
+								newCluster.add(Math.max(clique.get(i), clique.get(j)));
+								
+								int insertIndex = -1;
+								for(int k = 0; k < sequentialClusters.size(); k++)
+								{
+									if (newCluster.get(newCluster.size() - 1) < sequentialClusters.get(k).get(0))
+									{
+										insertIndex = k;
+										break;
+									}
+								}
+								
+								if (insertIndex >= 0) sequentialClusters.add(insertIndex, newCluster);
+								else sequentialClusters.add(newCluster);
+								
+								change = true;
+							}
+						}
+						
+						// one node is in one cluster, the other isn't
+						else
+						{
+							List<Integer> cluster = existingClusterFirst.isPresent() ? existingClusterFirst.get() : existingClusterSecond.get();
+							int node = existingClusterFirst.isPresent() ? clique.get(j) : clique.get(i);
+							
+							if ((node == cluster.get(0) - 1) || (node == cluster.get(cluster.size()-1) + 1))
+							{
+								cluster.add(node);
+								cluster.sort((e1, e2) -> e1 < e2 ? -1 : (e1 > e2 ? 1 : 0));
+								
+								change = true;
+							}
+						}						
+					}
+				}
+			}
+		}
+	}
+	
+	private List<Integer> computeSingletons(List<List<Integer>> cliques, List<List<Integer>> sequentialClusters)
+	{
+		List<Integer> singletons = new ArrayList<Integer>();
+		for(List<Integer> c : cliques)
+		{	
+			for(int n : c)
+			{
+				if (!sequentialClusters.stream().anyMatch(sc -> sc.contains(n))) singletons.add(n);
+			}
+		}
+		
+		singletons = singletons.stream().distinct().collect(Collectors.toList()); 
+		singletons.sort((s1, s2) -> s1 < s2 ? -1 : (s1 > s2 ? 1 : 0));
+		return singletons;
+	}
+	
+	private void mergeTooSmallSequences(List<List<Integer>> sequentialClusters, Map<Integer, Map<Integer, Double>> allSimilarities, int largestSmallCluster)
+	{
+		boolean change = true;
+		while(change)
+		{
+			change = false;
+			Optional<List<Integer>> firstSmallCluster = sequentialClusters.stream().filter(c -> c.size() <= largestSmallCluster).findFirst();
+			if (firstSmallCluster.isPresent())
+			{
+				int i = sequentialClusters.indexOf(firstSmallCluster.get());
+				double similarityPrevious = (i == 0) ? 0 : averageClusterSimilarity(sequentialClusters.get(i-1), sequentialClusters.get(i), allSimilarities);
+				double similarityNext = (i == (sequentialClusters.size() - 1)) ? 0 : averageClusterSimilarity(sequentialClusters.get(i), sequentialClusters.get(i+1), allSimilarities);
+				
+				List<Integer> clusterToMergeWith = (similarityPrevious > similarityNext) ? sequentialClusters.get(i-1) : sequentialClusters.get(i+1);
+				List<Integer> newCluster = new ArrayList<Integer>();
+				newCluster.addAll(clusterToMergeWith);
+				newCluster.addAll(sequentialClusters.get(i));
+				newCluster.sort((i1, i2) -> i1 > i2 ? 1 : (i1 < i2 ? -1 : 0));
+				
+				sequentialClusters.add((similarityPrevious > similarityNext) ? i-1 : i, newCluster);
+				sequentialClusters.remove(firstSmallCluster.get());
+				sequentialClusters.remove(clusterToMergeWith);
+				
+				change = true;
+			}
+		}
+	}
+	
+	private double averageClusterSimilarity(List<Integer> first, List<Integer> second,  Map<Integer, Map<Integer, Double>> allSimilarities)
+	{
+		double sum = 0;
+		for(int i = 0; i < first.size(); i++)
+		{
+			for(int j = 0; j < second.size(); j++)
+			{
+				sum += allSimilarities.get(Math.min(first.get(i), second.get(j))).get(Math.max(first.get(i), second.get(j)));
+			}
+		}
+		return sum / ((double)(first.size() * second.size()));
+	}
+	
+	private void mergeSingletons(List<List<Integer>> cliques, List<List<Integer>> sequentialClusters, Map<Integer, Map<Integer, Double>> allSimilarities)
+	{
+        List<Integer> singletons = computeSingletons(cliques, sequentialClusters); 
+		
+		while(singletons.size() > 0)
+		{
+			if (singletons.size() % 10 == 0) System.out.println("Remaining singletons: " + singletons.size());
+			
+			int node = singletons.get(0);
+			Optional<List<Integer>> previousNodeCluster = sequentialClusters.stream().filter(sc -> sc.contains(node - 1)).findFirst();
+			Optional<List<Integer>> nextNodeCluster = sequentialClusters.stream().filter(sc -> sc.contains(node + 1)).findFirst();
+			
+			double similarityPrevious = node == 0 ? -1.0 : (previousNodeCluster.isPresent() ? similarityNodeCluster(node, previousNodeCluster.get(), allSimilarities) : allSimilarities.get(node - 1).get(node));
+            double similarityNext = node == allSimilarities.size() ? -1.0 : (nextNodeCluster.isPresent() ? similarityNodeCluster(node, nextNodeCluster.get(), allSimilarities) : allSimilarities.get(node).get(node + 1));
+
+            boolean previous = similarityPrevious >= similarityNext;
+            boolean mergeWithCluster = previous ? previousNodeCluster.isPresent() : nextNodeCluster.isPresent();
+            
+            if (mergeWithCluster)
+            {
+            	if (previous) previousNodeCluster.get().add(node);
+            	else nextNodeCluster.get().add(0, node);
+            }
+            else
+            {
+            	List<Integer> newCluster = new ArrayList<Integer>();
+				newCluster.add(previous ? node - 1 : node);
+				newCluster.add(previous ? node : node + 1);
+				
+				int insertIndex = -1;
+				
+				for(int k = 0; k < sequentialClusters.size(); k++)
+				{
+					if (newCluster.get(newCluster.size() - 1) < sequentialClusters.get(k).get(0))
+					{
+						insertIndex = k;
+						break;
+					}
+				}
+				
+				if (insertIndex >= 0) sequentialClusters.add(insertIndex, newCluster);
+				else sequentialClusters.add(newCluster);
+            }
+            
+            singletons = computeSingletons(cliques, sequentialClusters);
+		}		
+	}
+	
+	private double similarityNodeCluster(int node, List<Integer> cluster, Map<Integer, Map<Integer, Double>> allSimilarities)
+	{
+		double average = 0;
+		for(Integer n2 : cluster) average += allSimilarities.get(Math.min(node,  n2)).get(Math.max(node,  n2));
+		return average /((double)cluster.size());
+	}
+}

source/src/edu/uma/nlp/graphseg/GraphHandler.java

@@ -0,0 +1,134 @@
+package edu.uma.nlp.graphseg;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.jgrapht.UndirectedGraph;
+import org.jgrapht.alg.BronKerboschCliqueFinder;
+import org.jgrapht.alg.ConnectivityInspector;
+import org.jgrapht.alg.KuhnMunkresMinimalWeightBipartitePerfectMatching;
+import org.jgrapht.generate.SimpleWeightedBipartiteGraphMatrixGenerator;
+import org.jgrapht.generate.WeightedGraphGeneratorAdapter;
+import org.jgrapht.graph.DefaultEdge;
+import org.jgrapht.graph.DefaultWeightedEdge;
+import org.jgrapht.graph.SimpleGraph;
+import org.jgrapht.graph.SimpleWeightedGraph;
+
+import edu.uma.nlp.graphseg.preprocessing.Document;
+import edu.uma.nlp.graphseg.preprocessing.TokenAnnotation;
+import edu.uma.nlp.graphseg.utils.MemoryStorage;
+
+public class GraphHandler {
+	
+	private static List<String> stopwords;
+	public static void setStopwords(List<String> stwrds)
+	{
+		stopwords = stwrds;
+	}
+	
+	private static Map<Integer, Map<Integer, Double>> allSimilarities;
+	public static Map<Integer, Map<Integer, Double>> getAllSimilarities()
+	{
+		return allSimilarities;
+	}
+	
+	public static UndirectedGraph<Integer, DefaultEdge> constructGraph(List<Document> snippets, double similarityTreshold)
+	{
+		int localizationSize = 100;
+		allSimilarities = new HashMap<Integer, Map<Integer, Double>>();
+		
+		UndirectedGraph<Integer, DefaultEdge> graph = new SimpleGraph<Integer, DefaultEdge>(DefaultEdge.class);
+		
+		snippets.forEach(s -> graph.addVertex(Integer.parseInt(s.getId())));
+		for(int i = 0; i < snippets.size() - 1; i++)
+		{
+			if (i % 10 == 0) System.out.println("Constructing graph, outer loop " + i + "/" + snippets.size());
+			allSimilarities.put(i, new HashMap<Integer, Double>());
+			
+			for(int j = i + 1; j < Math.min(snippets.size(), i + localizationSize); j++)
+			{
+				List<TokenAnnotation> contentTokenFirst = snippets.get(i).getTokens().stream().filter(t -> t.getPartOfSpeech().isContent() && !stopwords.contains(t.getLemma().toLowerCase())).collect(Collectors.toList());
+				List<TokenAnnotation> contentTokenSecond = snippets.get(j).getTokens().stream().filter(t -> t.getPartOfSpeech().isContent() && !stopwords.contains(t.getLemma().toLowerCase())).collect(Collectors.toList());
+				
+				if (contentTokenFirst.size() == 0 || contentTokenSecond.size() == 0) 
+				{
+					allSimilarities.get(i).put(j, 0.0);
+					continue;
+				}
+				
+				// preparing for bipartite graph min matching 
+				double[][] dissimilarities = new double[Math.max(contentTokenFirst.size(), contentTokenSecond.size())][Math.max(contentTokenFirst.size(), contentTokenSecond.size())];
+				List<Integer> firstPartition = new ArrayList<Integer>();
+				List<Integer> secondPartition = new ArrayList<Integer>();
+				
+				for(int k = 0; k < Math.max(contentTokenFirst.size(), contentTokenSecond.size()); k++)
+				{
+					for(int l = 0; l < Math.max(contentTokenFirst.size(), contentTokenSecond.size()); l++)
+					{
+						if (k >= contentTokenFirst.size() || l >= contentTokenSecond.size())
+						{
+							dissimilarities[k][l] = 1;
+						}
+						else
+						{
+							double icFactor = Math.max(MemoryStorage.getInformationContent().getRelativeInformationContent(contentTokenFirst.get(k).getLemma().toLowerCase()), MemoryStorage.getInformationContent().getRelativeInformationContent(contentTokenSecond.get(l).getLemma().toLowerCase()));
+							double simTokens = MemoryStorage.getWordVectorSpace().similarity(contentTokenFirst.get(k).getLemma().toLowerCase(), contentTokenSecond.get(l).getLemma().toLowerCase());
+							if (simTokens < 0) simTokens = 0;
+							
+							dissimilarities[k][l] = 1 - icFactor * simTokens;
+						}
+					}
+				}
+				for(int z = 0; z < Math.max(contentTokenFirst.size(), contentTokenSecond.size()); z++)
+				{
+					firstPartition.add(z);
+					secondPartition.add(z + Math.max(contentTokenFirst.size(), contentTokenSecond.size()));
+				}
+				
+				double bmScore = minimumAverageBipartiteGraphMatchingScore(dissimilarities, firstPartition, secondPartition) - (Math.abs(contentTokenFirst.size() - contentTokenSecond.size()));
+				double similarityNonNormalized = Math.min(contentTokenFirst.size(), contentTokenSecond.size()) - bmScore;
+				double similarity = ((similarityNonNormalized / contentTokenFirst.size()) + (similarityNonNormalized / contentTokenSecond.size())) / 2.0;
+				
+				//double similarity = SemanticSimilarity.greedyAlignmentOverlapFScore(snippets.get(i).getTokens(), snippets.get(j).getTokens(), MemoryStorage.getWordVectorSpace(), MemoryStorage.getInformationContent(), true);
+				allSimilarities.get(i).put(j, similarity);
+				
+				if (similarity > similarityTreshold)
+				{
+					graph.addEdge(Integer.parseInt(snippets.get(i).getId()), Integer.parseInt(snippets.get(j).getId()));
+				}
+			}
+		}
+		
+		return graph;
+	}
+	
+	public static double minimumAverageBipartiteGraphMatchingScore(double[][] dissimilarities, List<Integer> firstPartition, List<Integer> secondPartition)
+	{
+		SimpleWeightedGraph<Integer, DefaultWeightedEdge> bipartiteGraph = new SimpleWeightedGraph<>(DefaultWeightedEdge.class);
+		WeightedGraphGeneratorAdapter<Integer, DefaultWeightedEdge, Integer> generator = 
+		          new SimpleWeightedBipartiteGraphMatrixGenerator<Integer, DefaultWeightedEdge>() 
+		              .first  (firstPartition) 
+		              .second (secondPartition) 
+		              .weights(dissimilarities); 
+		
+		generator.generateGraph(bipartiteGraph, null, null);
+		KuhnMunkresMinimalWeightBipartitePerfectMatching<Integer, DefaultWeightedEdge> bipartiteMatching = new KuhnMunkresMinimalWeightBipartitePerfectMatching<Integer, DefaultWeightedEdge>(bipartiteGraph, firstPartition, secondPartition);
+		
+		return bipartiteMatching.getMatchingWeight();
+	}
+	
+	public static List<List<Integer>> getAllCliques(UndirectedGraph<Integer, DefaultEdge> graph)
+	{
+		BronKerboschCliqueFinder<Integer, DefaultEdge> finder = new BronKerboschCliqueFinder<Integer, DefaultEdge>(graph);
+		return finder.getAllMaximalCliques().stream().map(x -> x.stream().collect(Collectors.toList())).collect(Collectors.toList());
+	}
+	
+	public static List<List<Integer>> getAllConnectedComponents(UndirectedGraph<Integer, DefaultEdge> graph)
+	{
+		ConnectivityInspector<Integer, DefaultEdge> finder = new ConnectivityInspector<Integer, DefaultEdge>(graph);
+		return finder.connectedSets().stream().map(x -> x.stream().collect(Collectors.toList())).collect(Collectors.toList());
+	}
+}

source/src/edu/uma/nlp/graphseg/IOHandler.java

@@ -0,0 +1,33 @@
+package edu.uma.nlp.graphseg;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.util.List;
+
+public class IOHandler {
+	public static void writeSegmentation(List<String> rawLines, List<List<Integer>> segmentation, String path)
+	{
+		try {
+			File fout = new File(path);
+			FileOutputStream fos;
+			fos = new FileOutputStream(fout);
+			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos));
+			
+			for(int i = 0; i < segmentation.size(); i++)
+			{
+				for(int j = 0; j < segmentation.get(i).size(); j++)
+				{
+					bw.write(rawLines.get(segmentation.get(i).get(j)) + "\n");
+				}
+				bw.write("==========\n");
+			}
+			bw.close();
+		}
+        catch (Exception e) {
+		// TODO Auto-generated catch block
+        	e.printStackTrace();
+        }
+	}
+}

source/src/edu/uma/nlp/graphseg/STSHandler.java

@@ -0,0 +1,37 @@
+package edu.uma.nlp.graphseg;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.javatuples.Triplet;
+
+import edu.uma.nlp.graphseg.preprocessing.Document;
+import edu.uma.nlp.graphseg.semantics.InformationContent;
+import edu.uma.nlp.graphseg.semantics.SemanticSimilarity;
+import edu.uma.nlp.graphseg.semantics.WordVectorSpace;
+
+
+public class STSHandler {
+	public static List<Triplet<Document, Document, Double>> getSemanticSimilarities(List<Document> snippets, double simTreshold, WordVectorSpace vectorSpace, InformationContent informationContent)
+	{
+		List<Triplet<Document, Document, Double>> similarityGraph = new ArrayList<Triplet<Document, Document, Double>>();  
+				
+		for(int i = 0; i < snippets.size() - 1; i++)
+		{
+			System.out.println("Outer loop: " + String.valueOf(i+1) + "/" + String.valueOf(snippets.size() - 1));
+			for(int j = i + 1; j < snippets.size(); j++)
+			{
+				//if (j % 100 == 0) System.out.println("Inner loop: " + String.valueOf(j+1) + "/" + String.valueOf(snippets.size()));
+				
+				double similarity = SemanticSimilarity.greedyAlignmentOverlapFScore(snippets.get(i).getTokens(), snippets.get(j).getTokens(), vectorSpace, informationContent, true);
+				if (similarity > simTreshold)
+				{
+					similarityGraph.add(new Triplet<Document, Document, Double>(snippets.get(i), snippets.get(j), similarity));
+				}
+			}
+		}
+		
+		similarityGraph.sort((i1, i2) -> i1.getValue2() > i2.getValue2() ? -1 : (i1.getValue2() < i2.getValue2() ? 1 : 0));
+		return similarityGraph;
+	}
+}

source/src/edu/uma/nlp/graphseg/Start.java

@@ -0,0 +1,122 @@
+package edu.uma.nlp.graphseg;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.jgrapht.UndirectedGraph;
+import org.jgrapht.graph.DefaultEdge;
+
+import edu.uma.nlp.graphseg.preprocessing.Document;
+import edu.uma.nlp.graphseg.preprocessing.StanfordAnnotator;
+import edu.uma.nlp.graphseg.semantics.InformationContent;
+import edu.uma.nlp.graphseg.semantics.SemanticSimilarity;
+import edu.uma.nlp.graphseg.semantics.WordVectorSpace;
+import edu.uma.nlp.graphseg.utils.ApplicationConfiguration;
+import edu.uma.nlp.graphseg.utils.IOHelper;
+import edu.uma.nlp.graphseg.utils.MemoryStorage;
+
+public class Start {
+	
+	public static void main(String[] args) throws NumberFormatException, IOException {
+		// TODO Auto-generated method stub
+
+		// checking the arguments 
+		if (args.length < 4)
+		{
+			System.out.println("USAGE: java -jar graphseg.jar <input-dir> <output-dir> <rel-treshold> <min-segment>");
+			return;
+		}
+		
+		File inputDirFile = new File(args[0]);
+		File outputDirFile = new File(args[1]);
+		
+		if (!inputDirFile.exists() || !outputDirFile.exists() || !inputDirFile.isDirectory() || !outputDirFile.isDirectory())
+		{
+			System.out.println("USAGE: java -jar graphseg.jar <input-dir> <output-dir> <rel-treshold (double, <0,1>)> <min-segment (int)>");
+			return;
+		}
+		
+		double treshold = 0;
+		try
+		{
+			treshold = Double.parseDouble(args[2]);
+			if (treshold < 0 || treshold > 1) 
+			{
+				throw new UnsupportedOperationException();
+			}
+		}
+		catch(NumberFormatException ex)
+		{
+			System.out.println("USAGE: java -jar graphseg.jar <input-dir> <output-dir> <rel-treshold (double, <0,1>)> <min-segment (int)>");
+			return;
+		}
+		
+		int minseg = 0;
+		try
+		{
+			minseg = Integer.parseInt(args[3]);
+			if (minseg < 1) 
+			{
+				throw new UnsupportedOperationException();
+			}
+		}
+		catch(NumberFormatException ex)
+		{
+			System.out.println("USAGE: java -jar graphseg.jar <input-dir> <output-dir> <rel-treshold (double, <0,1>)> <min-segment (int, >=1)>");
+			return;
+		}
+		
+		List<String> stopwords = IOHelper.getAllLines(ApplicationConfiguration.config.getValue("stop-words-path"));
+		MemoryStorage.setWordVectorSpace(new WordVectorSpace());
+		MemoryStorage.getWordVectorSpace().load(ApplicationConfiguration.config.getValue("word-vec-path"), null);
+		
+		MemoryStorage.setInformationContent(new InformationContent(ApplicationConfiguration.config.getValue("inf-cont-path"), 1));
+		
+		
+		SemanticSimilarity.setStopwords(stopwords);
+		GraphHandler.setStopwords(stopwords);
+		
+		StanfordAnnotator annotator = new StanfordAnnotator();
+    	
+		
+    	for(Path file : Files.walk(Paths.get(args[0])).filter(x -> (new File(x.toString()).isFile())).collect(Collectors.toList()))
+    	{
+			System.out.println("Segmenting file: " + file.toString());
+			
+			annotator.setStanfordAnnotators(new ArrayList<String>(Arrays.asList("tokenize", "ssplit")));
+			
+			String content = FileUtils.readFileToString(new File(file.toString()));
+			Document doc = new Document();
+			doc.setText(content);
+			annotator.annotate(doc);
+			
+			annotator.setStanfordAnnotators(new ArrayList<String>(Arrays.asList("tokenize", "ssplit", "pos", "lemma")));
+			
+	    	List<Document> snippets = new ArrayList<Document>();
+	    	for(int i = 0; i < doc.getSentences().size(); i++)
+	    	{
+	    		Document snippet = new Document(doc.getSentences().get(i).getText());
+	    		annotator.annotate(snippet);
+	    		snippet.setId(String.valueOf(i));
+	    		snippets.add(snippet);
+	    	}
+	    	
+	    	UndirectedGraph<Integer, DefaultEdge> graph = GraphHandler.constructGraph(snippets, treshold);
+	    	System.out.println("Computing cliques..."); 
+	    	List<List<Integer>> cliques = GraphHandler.getAllCliques(graph);
+			
+	    	ClusteringHandler clusterer = new ClusteringHandler();
+	    	System.out.println("Constructing linear segments..."); 
+	    	List<List<Integer>> sequentialClusters = clusterer.getSequentialClusters(cliques, GraphHandler.getAllSimilarities(), minseg);
+	    	IOHandler.writeSegmentation(doc.getSentences().stream().map(x -> x.getText()).collect(Collectors.toList()), sequentialClusters, args[1] + (args[1].endsWith("/") ? "" : "/") + file.getFileName().toString());
+		}	
+	}
+}

source/src/edu/uma/nlp/graphseg/preprocessing/Annotation.java

@@ -0,0 +1,36 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class Annotation {
+	 
+     protected HashMap<AnnotationType, List<Annotation>> childAnnotations;
+
+     public Annotation()
+     {
+         childAnnotations = new HashMap<AnnotationType, List<Annotation>>();
+     }
+
+     public List<Annotation> getChildAnnotations(AnnotationType type)
+     {
+         if (childAnnotations.containsKey(type)) return childAnnotations.get(type);
+         else return new ArrayList<Annotation>();
+     }
+
+     public void addChildAnnotation(Annotation annotation, AnnotationType type)
+     {
+         if (!childAnnotations.containsKey(type)) childAnnotations.put(type, new ArrayList<Annotation>());
+         childAnnotations.get(type).add(annotation);
+     }
+     
+     public void removeChildAnnotation(Annotation annotation)
+     {
+    	 if (childAnnotations.containsKey(annotation))
+    	 {
+    		 childAnnotations.remove(annotation);
+    	 }
+     }
+
+}

source/src/edu/uma/nlp/graphseg/preprocessing/AnnotationType.java

@@ -0,0 +1,14 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+public enum AnnotationType {
+	Corpus,
+	Document,
+    SentenceAnnotation,
+    TokenAnnotation,
+    MorphologyAnnotation,
+    PartOfSpeechAnnotation,
+    NamedEntityAnnotation,
+    NamedEntityTokenAnnotation,
+    ChunkAnnotation,
+    DependencyAnnotation
+}

source/src/edu/uma/nlp/graphseg/preprocessing/AnnotatorChain.java

@@ -0,0 +1,35 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AnnotatorChain {
+
+    private List<IAnnotator> chain;
+
+    public AnnotatorChain()
+    {
+    }
+
+    
+    public AnnotatorChain(List<IAnnotator> annotators)
+    {
+        chain = annotators;
+    }
+
+    public AnnotatorChain addAnnotator(IAnnotator annotator)
+    {
+        if (chain == null) chain = new ArrayList<IAnnotator>();
+        chain.add(annotator);
+        return this;
+    }
+
+    public void apply(Annotation textUnit)
+    {
+        for (int i = 0; i < chain.size(); i++)
+        {
+            chain.get(i).annotate(textUnit);
+        }
+    }
+
+}

source/src/edu/uma/nlp/graphseg/preprocessing/AnnotatorType.java

@@ -0,0 +1,11 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+public enum AnnotatorType
+{
+    SentenceSplitter,
+    Tokenizer,
+    POSTagger,
+    Morphology,
+    NamedEntityExtractor,
+    Chunker
+}

source/src/edu/uma/nlp/graphseg/preprocessing/Document.java

@@ -0,0 +1,110 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class Document extends Annotation {
+	
+	private String id;
+    
+    public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	private String path;
+	
+	public String getPath() {
+		return path;
+	}
+
+	public void setPath(String path) {
+		this.path = path;
+	}
+
+	private String text;
+	
+	public String getText() {
+		return text;
+	}
+
+	public void setText(String text) {
+		this.text = text;
+	}
+
+
+	// Lazy loading
+
+    private List<TokenAnnotation> tokens;
+    
+    public List<TokenAnnotation> getTokens() {
+		if (tokens == null) tokens = loadTokens();
+    	return tokens;
+	}
+
+	private List<SentenceAnnotation> sentences;
+	
+    public List<SentenceAnnotation> getSentences() {
+		if (sentences == null) sentences = loadSentences();
+    	return sentences;
+	}
+
+	private List<NamedEntityAnnotation> namedEntities;
+	
+    public List<NamedEntityAnnotation> getNamedEntities() {
+    	if (namedEntities == null) namedEntities = loadNamedEntities();
+    	return namedEntities;
+	}
+
+	private List<TokenAnnotation> loadTokens()
+    {
+        if (getSentences() != null)
+        {
+            List<TokenAnnotation> toks = new ArrayList<TokenAnnotation>();
+            for (int i = 0; i < sentences.size(); i++)
+            {
+                toks.addAll(sentences.get(i).getTokens());
+            }
+            
+            toks.sort((t1, t2) -> (t1.getStartPosition() < t2.getStartPosition()) ? -1 : ((t1.getStartPosition() > t2.getStartPosition()) ? 1 : 0));
+            return toks;
+        }
+        else return null;
+
+    }
+
+    private List<SentenceAnnotation> loadSentences()
+    {	
+        if (childAnnotations.containsKey(AnnotationType.SentenceAnnotation)) 
+        	return childAnnotations.get(AnnotationType.SentenceAnnotation).stream().map(x -> (SentenceAnnotation)x).collect(Collectors.toList());
+        else return null;
+    }
+
+    private List<NamedEntityAnnotation> loadNamedEntities()
+    {
+        if (childAnnotations.containsKey(AnnotationType.NamedEntityAnnotation)) 
+        	return childAnnotations.get(AnnotationType.NamedEntityAnnotation).stream().map(x -> (NamedEntityAnnotation)x).collect(Collectors.toList());
+        else return null;
+    }
+    
+    // Ctors
+    
+    public Document()
+    {
+    }
+    
+    public Document(String text)
+    {
+    	this.text = text;
+    }
+    
+    public Document(String id, String text)
+    {
+    	this.id = id;
+    	this.text = text;
+    }
+}

source/src/edu/uma/nlp/graphseg/preprocessing/IAnnotator.java

@@ -0,0 +1,9 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.List;
+
+public interface IAnnotator
+{
+    void annotate(Annotation textUnit);
+    List<Annotation> annotate(String text);
+}

source/src/edu/uma/nlp/graphseg/preprocessing/NamedEntityAnnotation.java

@@ -0,0 +1,88 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class NamedEntityAnnotation extends Annotation {
+
+	private NamedEntityType namedEntityType;
+
+    public NamedEntityType getNamedEntityType() {
+		return namedEntityType;
+	}
+
+	public void setNamedEntityType(NamedEntityType namedEntityType) {
+		this.namedEntityType = namedEntityType;
+	}
+
+    private String text;
+    
+	public String getText() {
+		if ((text == null || StringUtils.isEmpty(text)) && getTokens().size() > 0)
+		{
+			text = "";
+			for(int i = 0; i < tokens.size(); i++)
+			{
+				text += tokens.get(i).getText() + " ";
+			}
+			text = text.trim();
+		}
+		return text;
+	}
+
+	public void setText(String text) {
+		this.text = text;
+	}
+    
+    private int startPosition;
+    
+	public int getStartPosition() {
+		return startPosition;
+	}
+
+	public void setStartPosition(int startPosition) {
+		this.startPosition = startPosition;
+	}
+
+	public Boolean isPerson()
+    {       
+        return namedEntityType == NamedEntityType.Person;
+    }
+
+	public Boolean isLocation()
+    {       
+        return namedEntityType == NamedEntityType.Location;
+    }
+
+	public Boolean isOrganization()
+    {       
+        return namedEntityType == NamedEntityType.Organization;
+    }
+
+    private List<TokenAnnotation> tokens;
+    public List<TokenAnnotation> getTokens()
+    {
+    	if (tokens == null) tokens = loadTokens();
+    	return tokens;
+    }
+    
+    private List<TokenAnnotation> loadTokens()
+    {
+        if (childAnnotations.containsKey(AnnotationType.TokenAnnotation)) return childAnnotations.get(AnnotationType.TokenAnnotation).stream().map(x -> (TokenAnnotation)x).collect(Collectors.toList());
+        else return null;
+    }
+    
+    public NamedEntityAnnotation(NamedEntityType type)
+    {
+        namedEntityType = type;
+    }
+    
+    public NamedEntityAnnotation(NamedEntityType type, String text, int startPosition)
+    {
+    	this.namedEntityType = type;
+    	this.text = text;
+    	this.startPosition = startPosition;
+    }
+}

source/src/edu/uma/nlp/graphseg/preprocessing/NamedEntityTokenAnnotation.java

@@ -0,0 +1,38 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+public class NamedEntityTokenAnnotation extends Annotation {
+	
+	private String namedEntityLabel;
+	   
+    public String getNamedEntityLabel() {
+		return namedEntityLabel;
+	}
+
+	public void setNamedEntityLabel(String namedEntityLabel) {
+		this.namedEntityLabel = namedEntityLabel;
+	}
+
+	public NamedEntityTokenAnnotation()
+    {
+    }
+
+    public NamedEntityTokenAnnotation(String label)
+    {
+        namedEntityLabel = label;
+    }
+
+    public Boolean constitutesNamedEntity()
+    {
+        return startsNamedEntity() || insideNamedEntity();
+    }
+
+    public Boolean startsNamedEntity()
+    {
+        return namedEntityLabel.startsWith("B");
+    }
+
+    public Boolean insideNamedEntity()
+    {
+        return namedEntityLabel.startsWith("I");
+    }
+}

source/src/edu/uma/nlp/graphseg/preprocessing/NamedEntityType.java

@@ -0,0 +1,18 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+public enum NamedEntityType {
+	Person,
+	Location,
+	Organization,
+	Money,
+	Percentage,
+	Date,
+	Time,
+	Ordinal,
+	Percent,
+	Number,
+	Set,
+	Duration,
+	Misc,
+	None
+}

source/src/edu/uma/nlp/graphseg/preprocessing/PartOfSpeechAnnotation.java

@@ -0,0 +1,69 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class PartOfSpeechAnnotation extends Annotation {
+	
+    private String tag;
+
+    public String getTag() {
+		return tag;
+	}
+
+	public void setTag(String tag) {
+		this.tag = tag;
+	}
+	
+	private String chunkTag;
+	
+	public String getChunkTag() {
+		return chunkTag;
+	}
+
+	public void setChunkTag(String chunkTag) {
+		this.chunkTag = chunkTag;
+	}
+
+	private String coarseTag;
+	public String getCoarseTag() {
+		if (tag != null) coarseTag = loadCoarsePoSTag();
+		return coarseTag;			
+	}
+
+    private String loadCoarsePoSTag()
+    {
+        if (isNoun()) return "N";
+        else if (isVerb()) return "V";
+        else if (isAdjective()) return "J";
+        else if (isAdverb()) return "R";
+        else return "O";
+    }
+
+    public Boolean isContent()
+    {
+    	List<String> otherContentPOS = Arrays.asList("CD", "FW", "MD", "SYM", "UH");
+        return isNoun() || isVerb() || isAdjective() || isAdverb() || otherContentPOS.contains(tag);
+    }
+    
+    public Boolean isNoun()
+    {
+        return tag != null && tag.startsWith("N");
+    }
+
+    public Boolean isVerb()
+    {
+        return tag != null && tag.startsWith("V");
+    }
+
+    public Boolean isAdjective()
+    {
+        return tag != null && tag.startsWith("J");
+    }
+
+    public Boolean isAdverb()
+    {
+    	return tag != null && tag.startsWith("RB");
+    }
+    
+}

source/src/edu/uma/nlp/graphseg/preprocessing/SentenceAnnotation.java

@@ -0,0 +1,66 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class SentenceAnnotation extends Annotation {
+
+	// Fields & properties
+	
+    private String text;
+
+    public String getText() {
+		return text;
+	}
+
+	public void setText(String text) {
+		this.text = text;
+	}
+
+	private int startPosition;
+	
+	public int getStartPosition() {
+		return startPosition;
+	}
+
+	public void setStartPosition(int startPosition) {
+		this.startPosition = startPosition;
+	}
+
+    public int getEndPosition()
+    { 
+    	return text != null ? startPosition + text.length() - 1 : startPosition;
+    }
+    
+    // Lazy properties
+    
+    private List<TokenAnnotation> tokens;
+    public List<TokenAnnotation> getTokens()
+    {
+    	if (tokens == null) tokens = loadTokens();
+    	return tokens;
+    }
+    
+    public void setTokens(List<TokenAnnotation> tokens)
+    {
+    	this.tokens = tokens;
+    }
+    
+    private List<TokenAnnotation> loadTokens()
+    {
+        if (childAnnotations.containsKey(AnnotationType.TokenAnnotation)) return childAnnotations.get(AnnotationType.TokenAnnotation).stream().map(x -> (TokenAnnotation)x).collect(Collectors.toList());
+        else return null;
+    }
+
+
+    // Ctors
+    public SentenceAnnotation()
+    {
+    }
+    
+    public SentenceAnnotation(String text, int startPosition)
+    {
+    	this.text = text;
+    	this.startPosition = startPosition;
+    }
+}

source/src/edu/uma/nlp/graphseg/preprocessing/StanfordAnnotator.java

@@ -0,0 +1,142 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import edu.stanford.nlp.ling.CoreAnnotations.CharacterOffsetBeginAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.SentencesAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
+import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
+import edu.stanford.nlp.ling.CoreLabel;
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+import edu.stanford.nlp.util.CoreMap;
+
+public class StanfordAnnotator implements IAnnotator {
+
+	private List<String> stanfordAnnotators;
+	private String stanfordAnnotatorsString;
+
+	public void setStanfordAnnotators(List<String> stanfordAnnotators) {
+		this.stanfordAnnotators = stanfordAnnotators;
+		
+		stanfordAnnotatorsString = "";
+		for(int i = 0; i < this.stanfordAnnotators.size(); i++)
+		{
+			if (i == 0) stanfordAnnotatorsString += this.stanfordAnnotators.get(i);
+			else stanfordAnnotatorsString += ", " + this.stanfordAnnotators.get(i);
+		}
+	}
+
+	@Override
+	public void annotate(Annotation textUnit) 
+	{
+		if (textUnit instanceof Document)
+		{
+			Properties props = new Properties();
+		    props.setProperty("annotators", stanfordAnnotatorsString);
+		    
+		    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+		    
+		    edu.stanford.nlp.pipeline.Annotation docAnnotation = new edu.stanford.nlp.pipeline.Annotation(((Document)textUnit).getText());
+		    pipeline.annotate(docAnnotation);
+		    
+		    List<CoreMap> sentences = docAnnotation.get(SentencesAnnotation.class);
+		    
+		    for(CoreMap stanfordSentence : sentences) 
+		    {
+			      SentenceAnnotation sentence = new SentenceAnnotation();
+			      sentence.setText(stanfordSentence.get(TextAnnotation.class));
+			      sentence.setStartPosition(stanfordSentence.get(CharacterOffsetBeginAnnotation.class));
+			      
+			      for (CoreLabel stanfordToken: stanfordSentence.get(TokensAnnotation.class)) 
+			      {
+			    	  	TokenAnnotation token = new TokenAnnotation(stanfordToken.get(TextAnnotation.class));
+				        token.setStartPosition(stanfordToken.beginPosition());
+				        token.setSentenceIndex(stanfordToken.sentIndex());
+				        
+				        if (stanfordAnnotators.contains("lemma"))
+				        {
+				        	token.setLemma(stanfordToken.lemma());		       
+				        }
+				        
+				        if (stanfordAnnotators.contains("pos"))
+				        {
+					        PartOfSpeechAnnotation posAnnotation = new PartOfSpeechAnnotation();
+					        posAnnotation.setTag(stanfordToken.get(edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation.class));
+					        token.addChildAnnotation(posAnnotation, AnnotationType.PartOfSpeechAnnotation);
+				        }
+				        
+				        if (stanfordAnnotators.contains("ner"))
+				        {
+				        	NamedEntityTokenAnnotation neta = new NamedEntityTokenAnnotation(stanfordToken.get(NamedEntityTagAnnotation.class));
+				        	token.addChildAnnotation(neta, AnnotationType.NamedEntityTokenAnnotation);
+				        }
+				        
+				        sentence.addChildAnnotation(token, AnnotationType.TokenAnnotation);
+			    }
+		        
+		      	// linking continuous token-level NE annotations into whole named entity annotations
+		      	if (stanfordAnnotators.contains("ner"))
+		        {
+		      		List<NamedEntityAnnotation> nes = new ArrayList<NamedEntityAnnotation>();
+		      		NamedEntityAnnotation ne = null;
+		      		for(int i = 0; i < sentence.getTokens().size(); i++)
+		      		{
+		      			String neLabel = sentence.getTokens().get(i).getNamedEntityLabel().getNamedEntityLabel();
+		      			String neLabelPrevious = i > 0 ? sentence.getTokens().get(i-1).getNamedEntityLabel().getNamedEntityLabel() : "O";
+		      			
+		      			if (neLabel.compareTo("O") == 0)
+		      			{
+		      				if (ne != null)
+		      				{
+		      					nes.add(ne);
+		      					ne = null;
+		      				}
+		      			}
+		      			else if (neLabel.compareTo(neLabelPrevious) != 0)
+		      			{	
+      						NamedEntityType type = Arrays.stream(NamedEntityType.values()).filter(e -> e.name().equalsIgnoreCase(neLabel)).findAny().orElse(null);
+      						if (type == null)
+      						{
+      							throw new UnsupportedOperationException("Unknown named entity type!");
+      						}
+      					
+      						ne = new NamedEntityAnnotation(type);
+      						ne.setStartPosition(sentence.getTokens().get(i).getStartPosition());
+      						ne.addChildAnnotation(sentence.getTokens().get(i), AnnotationType.TokenAnnotation);
+      						
+		      			}
+		      			else
+		      			{
+		      				ne.addChildAnnotation(sentence.getTokens().get(i), AnnotationType.TokenAnnotation);
+		      			}
+		      		}
+		      		if (ne != null) nes.add(ne);
+		      		
+		      		nes.forEach(n -> textUnit.addChildAnnotation(n, AnnotationType.NamedEntityAnnotation));      		
+		        }
+		     
+			    textUnit.addChildAnnotation(sentence, AnnotationType.SentenceAnnotation);
+		    }
+		    
+		    // coreference, crosses sentence borders
+	        if (stanfordAnnotators.contains("dcoref"))
+	        {
+	        	// TODO: coref annotations
+	        }
+		}
+		else throw new UnsupportedOperationException("Only whole documents can be processed by Stanford's CoreNLP pipeline");
+	}
+
+	@Override
+	public List<Annotation> annotate(String text) 
+	{
+		Document document = new Document(text);
+		annotate(document);
+		
+		return new ArrayList<Annotation>(Arrays.asList(document));
+	}
+}

source/src/edu/uma/nlp/graphseg/preprocessing/TokenAnnotation.java

@@ -0,0 +1,104 @@
+package edu.uma.nlp.graphseg.preprocessing;
+
+public class TokenAnnotation extends Annotation {
+	
+	// Fields & properties
+	
+    private String text; 
+    
+    public String getText() {
+		return text;
+	}
+	public void setText(String text) {
+		this.text = text;
+	}
+	
+	private String lemma;
+	
+	public String getLemma() {
+		return lemma;
+	}
+	public void setLemma(String lemma) {
+		this.lemma = lemma;
+	}
+	
+	private int startPosition;
+	
+	public int getStartPosition() {
+		return startPosition;
+	}
+
+	public void setStartPosition(int startPosition) {
+		this.startPosition = startPosition;
+	}
+    
+	public int getEndPosition() {
+		return text != null ? startPosition + text.length() - 1 : startPosition;
+	}
+	
+    private int startPositionSentence;
+
+    public int getStartPositionSentence() {
+		return startPositionSentence;
+	}
+
+	public void setStartPositionSentence(int startPositionSentence) {
+		this.startPositionSentence = startPositionSentence;
+	}
+	
+	public int getEndPositionSentence() {
+		return text != null ? startPositionSentence + text.length() - 1 : startPositionSentence;
+	}
+	
+    private int sentenceIndex;
+    
+    public int getSentenceIndex() {
+		return sentenceIndex;
+	}
+
+	public void setSentenceIndex(int sentenceIndex) {
+		this.sentenceIndex = sentenceIndex;
+	}
+	
+	// Lazy loading properties
+   
+    private PartOfSpeechAnnotation partOfSpeech;
+	public PartOfSpeechAnnotation getPartOfSpeech()
+    {
+    	if (partOfSpeech == null) partOfSpeech = loadPartOfSpeech();
+    	return partOfSpeech;
+    }
+    
+
+    private NamedEntityTokenAnnotation namedEntityLabel;
+    public NamedEntityTokenAnnotation getNamedEntityLabel()
+    {
+        if (namedEntityLabel == null) namedEntityLabel = loadTokenNELabel();
+        return namedEntityLabel;
+    }
+
+    private PartOfSpeechAnnotation loadPartOfSpeech()
+    {
+        if (!childAnnotations.containsKey(AnnotationType.PartOfSpeechAnnotation)) this.addChildAnnotation(new PartOfSpeechAnnotation(), AnnotationType.PartOfSpeechAnnotation); 
+    	return ((PartOfSpeechAnnotation)(getChildAnnotations(AnnotationType.PartOfSpeechAnnotation).get(0)));
+    }
+
+    private NamedEntityTokenAnnotation loadTokenNELabel()
+    {
+        if (!childAnnotations.containsKey(AnnotationType.NamedEntityTokenAnnotation)) return null; //this.addChildAnnotation(new NamedEntityTokenAnnotation(), AnnotationType.NamedEntityTokenAnnotation); 
+        else return ((NamedEntityTokenAnnotation)(childAnnotations.get(AnnotationType.NamedEntityTokenAnnotation).get(0)));       
+    }
+    
+    public TokenAnnotation(String text, int startPosition, int startPositionSentence, int sentenceIndex)
+    {
+    	this.text = text;
+    	this.startPosition = startPosition;
+    	this.startPositionSentence = startPositionSentence;
+    	this.sentenceIndex = sentenceIndex;
+    }
+    
+    public TokenAnnotation(String text)
+    {
+    	this.text = text;
+    }
+}

source/src/edu/uma/nlp/graphseg/semantics/InformationContent.java

@@ -0,0 +1,77 @@
+package edu.uma.nlp.graphseg.semantics;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.List;
+
+import edu.uma.nlp.graphseg.utils.IOHelper;
+
+public class InformationContent {
+	private HashMap<String, Integer> frequencies = new HashMap<String, Integer>();
+    private double sumFrequencies = 0;
+    private double minFreq = 1;
+    private double divideFactor = 1; 
+
+    public InformationContent(String path, double divideFactor)
+    {
+        this.divideFactor = divideFactor;
+
+        frequencies = IOHelper.loadCounts(path);
+        sumFrequencies = frequencies.values().stream().mapToDouble(x -> ((double)x)/divideFactor).sum();
+        minFreq = frequencies.values().stream().mapToDouble(x -> ((double)x)/divideFactor).min().getAsDouble();
+        if (minFreq == 0) minFreq = 1;
+    }
+    
+    public InformationContent(InputStream stream, double divideFactor) throws UnsupportedEncodingException, IOException
+    {
+        this.divideFactor = divideFactor;
+
+        frequencies = IOHelper.loadCounts(stream);
+        sumFrequencies = frequencies.values().stream().mapToDouble(x -> ((double)x)/divideFactor).sum();
+        minFreq = frequencies.values().stream().mapToDouble(x -> ((double)x)/divideFactor).min().getAsDouble();
+        if (minFreq == 0) minFreq = 1;
+    }
+
+    public InformationContent(HashMap<String, Integer> frequenciesDictionary, double divideFactor)
+    {
+        this.divideFactor = divideFactor;
+
+        frequencies = frequenciesDictionary;
+        sumFrequencies = frequencies.values().stream().mapToDouble(x -> ((double)x)/divideFactor).sum();
+        minFreq = frequencies.values().stream().mapToDouble(x -> ((double)x)/divideFactor).min().getAsDouble();
+    }
+
+    public double getInformationContent(String word)
+    {
+        if (frequencies.containsKey(word.toLowerCase())) return (-1) * Math.log(((((double)frequencies.get(word.toLowerCase())) + minFreq) / divideFactor) / sumFrequencies);
+        else return (-1) * Math.log((minFreq / divideFactor) / sumFrequencies);
+    }
+
+    public double getRelativeInformationContent(String word)
+    {
+        double maxInfCont = (-1) * Math.log((minFreq / divideFactor) / sumFrequencies);
+        double infCont = (frequencies.containsKey(word.toLowerCase())) ? (-1) * Math.log(((((double)frequencies.get(word.toLowerCase())) + minFreq) / divideFactor) / sumFrequencies) : maxInfCont;
+
+        return infCont / maxInfCont;
+    }
+
+    public double getLogRelativeInformationContent(String word)
+    {
+        double maxInfCont = (-1) * Math.log((minFreq / divideFactor) / sumFrequencies);
+        double infCont = (frequencies.containsKey(word.toLowerCase())) ? (-1) * Math.log(((((double)frequencies.get(word.toLowerCase())) + minFreq) / divideFactor) / sumFrequencies) : maxInfCont;
+
+        return Math.log(infCont) / Math.log(maxInfCont);
+    }
+
+    public double getInformationContent(List<String> phrase)
+    {
+        double ic = 1;
+        for(String w : phrase)
+    	{
+        	ic *= getInformationContent(w);    	
+    	}
+        return ic;
+    }
+}

source/src/edu/uma/nlp/graphseg/semantics/SemanticSimilarity.java

@@ -0,0 +1,252 @@
+package edu.uma.nlp.graphseg.semantics;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.javatuples.Triplet;
+
+import edu.uma.nlp.graphseg.preprocessing.TokenAnnotation;
+import edu.uma.nlp.graphseg.utils.VectorOperations;
+
+public class SemanticSimilarity {
+	
+	private static List<String> stopwords;
+	public static void setStopwords(List<String> stwrds)
+	{
+		stopwords = stwrds;
+	}
+	
+	public static double greedyAlignmentOverlapFScore(List<TokenAnnotation> firstPhrase, List<TokenAnnotation> secondPhrase, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+        return greedyAlignmentOverlap(firstPhrase, secondPhrase, vectorSpace, informationContent, contentWordsOnly).getValue2();
+    }
+
+    public static double greedyAlignmentOverlapPrecision(List<TokenAnnotation> firstPhrase, List<TokenAnnotation> secondPhrase, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+        return greedyAlignmentOverlap(firstPhrase, secondPhrase, vectorSpace, informationContent, contentWordsOnly).getValue0();
+    }
+
+    public static double greedyAlignmentOverlapRecall(List<TokenAnnotation> firstPhrase, List<TokenAnnotation> secondPhrase, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+        return greedyAlignmentOverlap(firstPhrase, secondPhrase, vectorSpace, informationContent, contentWordsOnly).getValue1();
+    }
+
+    public static Triplet<Double, Double, Double> greedyAlignmentOverlap(List<TokenAnnotation> firstPhrase, List<TokenAnnotation> secondPhrase, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+        List<TokenAnnotation> firstPhraseCopy = new ArrayList<TokenAnnotation>();
+        List<TokenAnnotation> secondPhraseCopy = new ArrayList<TokenAnnotation>();
+        if (contentWordsOnly)
+        {
+            firstPhraseCopy.addAll(firstPhrase.stream().filter(x -> (contentWordsOnly ? x.getPartOfSpeech().isContent() : 1 != 2)).collect(Collectors.toList()));
+            secondPhraseCopy.addAll(secondPhrase.stream().filter(x -> (contentWordsOnly ? x.getPartOfSpeech().isContent() : 1 != 2)).collect(Collectors.toList()));
+        }
+        else
+        {
+            firstPhraseCopy.addAll(firstPhrase);
+            secondPhraseCopy.addAll(secondPhrase);
+        }
+        
+        if (stopwords != null && stopwords.size() > 0)
+        {
+        	firstPhraseCopy = firstPhraseCopy.stream().filter(t -> !stopwords.contains(t.getLemma().toLowerCase()) && !stopwords.contains(t.getText().toLowerCase())).collect(Collectors.toList());
+        	secondPhraseCopy = secondPhraseCopy.stream().filter(t -> !stopwords.contains(t.getLemma().toLowerCase()) && !stopwords.contains(t.getText().toLowerCase())).collect(Collectors.toList());
+        }
+
+        List<Double> pairSimilarities = new ArrayList<Double>(); 
+        while(firstPhraseCopy.size() > 0 && secondPhraseCopy.size() > 0)
+        {
+            double maxSim = -1;
+            TokenAnnotation firstToken = null;
+            TokenAnnotation secondToken = null;
+            for(TokenAnnotation nf : firstPhraseCopy)
+            {
+                for(TokenAnnotation ns : secondPhraseCopy)
+                {
+                    double sim = vectorSpace.similarity(nf.getText().toLowerCase(), ns.getText().toLowerCase());
+                    if (sim < 0) sim = 0;
+
+                    if (sim > maxSim)
+                    {
+                        firstToken = nf;
+                        secondToken = ns;
+                        maxSim = sim;
+                    }
+                }
+            }
+
+            if (informationContent != null) 
+        	{
+            		pairSimilarities.add(maxSim * Math.max(informationContent.getInformationContent(firstToken.getText().toLowerCase()), informationContent.getInformationContent(secondToken.getText().toLowerCase())));
+        	}
+            else pairSimilarities.add(maxSim);
+
+            firstPhraseCopy.remove(firstToken);
+            secondPhraseCopy.remove(secondToken);
+        }
+
+        double precision = 0;
+        double recall = 0;
+        double overlap = pairSimilarities.stream().mapToDouble(s -> s).sum();
+
+        if (informationContent != null)
+        {
+            double infContentFirst = contentWordsOnly ? 
+            	   firstPhrase.stream().filter(x -> x.getPartOfSpeech().isContent()).mapToDouble(t -> informationContent.getInformationContent(t.getText().toLowerCase())).sum() : 
+            	   firstPhrase.stream().mapToDouble(t -> informationContent.getInformationContent(t.getText().toLowerCase())).sum();
+            	   
+    	    double infContentSecond = contentWordsOnly ? 
+            	   secondPhrase.stream().filter(x -> x.getPartOfSpeech().isContent()).mapToDouble(t -> informationContent.getInformationContent(t.getText().toLowerCase())).sum() : 
+            	   secondPhrase.stream().mapToDouble(t -> informationContent.getInformationContent(t.getText().toLowerCase())).sum();
+            
+            precision = overlap / infContentFirst;
+            recall = overlap / infContentSecond;
+        }
+        else
+        {
+            precision = overlap / firstPhrase.size();
+            recall = overlap / secondPhrase.size();
+        }
+        
+        double fScore = 0;
+        if (precision == 0 && recall == 0) fScore = 0;
+        else fScore = (2 * precision * recall) / (precision + recall);
+        if (Double.isNaN(fScore)) fScore = 0;
+
+        return new Triplet<Double, Double, Double>(precision, recall, fScore);
+    }
+    
+    public static double embeddingSumSimilarity(List<TokenAnnotation> first, List<TokenAnnotation> second, WordVectorSpace vectorSpace, int embeddingLength, Boolean content, List<InformationContent> infContents)
+    {
+        double[] embeddingFirst = new double[embeddingLength];
+        double[] embeddingSecond = new double[embeddingLength];
+
+        if (content)
+        {
+            first = first.stream().filter(x -> x.getPartOfSpeech().isContent()).collect(Collectors.toList());
+            second = second.stream().filter(x -> x.getPartOfSpeech().isContent()).collect(Collectors.toList());
+        }
+
+        first.forEach(x -> 
+        {
+            double[] wordEmbedding = vectorSpace.getEmbedding(x.getText().trim());
+            if (wordEmbedding == null) 
+        	{
+            	wordEmbedding = vectorSpace.getEmbedding(x.getText().trim().toLowerCase());
+        	}
+            if (wordEmbedding != null)
+            {
+                double ic = 1;
+                for(InformationContent inco : infContents)
+                {
+                	ic *= inco.getInformationContent(x.getText().trim().toLowerCase());
+                };
+                VectorOperations.multiply(wordEmbedding, ic);
+                VectorOperations.addVector(embeddingFirst, wordEmbedding);
+            }
+        });
+
+        second.forEach(x ->
+        {
+            double[] wordEmbedding = vectorSpace.getEmbedding(x.getText().trim());
+            if (wordEmbedding == null)
+        	{
+            	wordEmbedding = vectorSpace.getEmbedding(x.getText().trim().toLowerCase());
+        	}
+            if (wordEmbedding != null)
+            {
+                double ic = 1;
+                for(InformationContent inco : infContents)
+                {
+                	ic *= inco.getInformationContent(x.getText().trim().toLowerCase());
+                };
+                VectorOperations.multiply(wordEmbedding, ic);
+                VectorOperations.addVector(embeddingSecond, wordEmbedding);
+            }
+        });
+
+        double res;
+		try {
+			res = VectorOperations.cosine(embeddingFirst, embeddingSecond);
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			res = 0;
+		}
+        if (Double.isNaN(res))
+        {
+            res = 0;
+        }
+        
+        return res;
+    }
+    
+    public static double averagePhraseGreedyAlignmentOverlap(List<List<TokenAnnotation>> firstPhrases, List<List<TokenAnnotation>> secondPhrases, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+    	double sum = 0;
+    	double counter = 0;
+    	
+    	for(List<TokenAnnotation> fp : firstPhrases){
+    		for(List<TokenAnnotation> sp : secondPhrases){
+    			double sim = greedyAlignmentOverlapFScore(fp, sp, vectorSpace, informationContent, contentWordsOnly);
+    			sum += sim;
+    			counter++;
+    		}
+    	}
+    	
+    	double score = sum/counter;
+    	if (Double.isNaN(score) || Double.isInfinite(score)) return 0;
+    	else return score;
+    }
+    
+    public static double maxPhraseGreedyAlignmentOverlap(List<List<TokenAnnotation>> firstPhrases, List<List<TokenAnnotation>> secondPhrases, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+    	double maxSim = 0;
+    	
+    	for(List<TokenAnnotation> fp : firstPhrases){
+    		for(List<TokenAnnotation> sp : secondPhrases){
+    			double sim = greedyAlignmentOverlapFScore(fp, sp, vectorSpace, informationContent, contentWordsOnly);
+    			if (sim > maxSim)
+    			{
+    				maxSim = sim;
+    			}
+    		}
+    	}   	
+    	return maxSim;
+    }
+    
+    public static int numSufficientlySimilarPhrasesGreedyAlignmentOverlap(List<List<TokenAnnotation>> firstPhrases, List<List<TokenAnnotation>> secondPhrases, double treshold, WordVectorSpace vectorSpace, InformationContent informationContent, Boolean contentWordsOnly)
+    {
+    	int counter = 0;
+    	
+    	for(List<TokenAnnotation> fp : firstPhrases){
+    		for(List<TokenAnnotation> sp : secondPhrases){
+    			double sim = greedyAlignmentOverlapFScore(fp, sp, vectorSpace, informationContent, contentWordsOnly);
+    			if (sim >= treshold)
+    			{
+    				counter++;
+    			}
+    		}
+    	}   	
+    	return counter;
+    }
+    
+    public static HashMap<String, Double> allToAllSimilarity(WordVectorSpace vectorSpace, List<String> vocabulary)
+    {
+    	HashMap<String, Double> similarities = new HashMap<String, Double>();
+    	for(int i = 0; i < vocabulary.size() - 1; i++)
+    	{
+    		if (i % 100 == 0) System.out.println("Outer loop: " + String.valueOf(i + 1) + "/" + String.valueOf(vocabulary.size() - 1));
+    		for(int j = i+1; j < vocabulary.size(); j++)
+    		{
+    			double sim = vectorSpace.similarity(vocabulary.get(i), vocabulary.get(j));
+    			if (sim >= -1)
+    			{
+    				similarities.put(vocabulary.get(i).compareTo(vocabulary.get(j)) < 0 ? vocabulary.get(i) + "<=>" + vocabulary.get(j) : vocabulary.get(j) + "<=>" + vocabulary.get(i), sim);
+    			}
+    		}
+    	}
+    	return similarities;
+    }
+}

source/src/edu/uma/nlp/graphseg/semantics/WordVectorSpace.java

@@ -0,0 +1,151 @@
+package edu.uma.nlp.graphseg.semantics;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.javatuples.Pair;
+
+import edu.uma.nlp.graphseg.utils.VectorOperations;
+
+public class WordVectorSpace {
+	
+	private HashMap<String, double[]> embeddings;
+    private int dimension;
+    
+    public int getDimension() {
+		return dimension;
+	}
+
+    public void load(String path, HashMap<String, Integer> filters) throws FileNotFoundException, IOException
+    {
+        embeddings = new HashMap<String, double[]>();
+
+        try (BufferedReader br = new BufferedReader(new FileReader(path))) {
+            String line;
+            int counter = 0;
+            while ((line = br.readLine()) != null) {
+            	try
+                {
+                    String split[] = line.trim().split("\\s+");
+
+                    if (filters == null || filters.containsKey(split[0].toLowerCase()))
+                    {
+                        dimension = split.length - 1;
+
+                        if (!embeddings.containsKey(split[0])) embeddings.put(split[0], new double[split.length - 1]);
+                        for (int i = 1; i < split.length; i++)
+                        {
+                            embeddings.get(split[0])[i - 1] = Double.parseDouble(split[i]);
+                        }
+                    }
+                    counter++;
+                    if (counter % 1000 == 0) 
+                    {
+                    	System.out.println("Loading vectors... " + String.valueOf(counter));
+                	}
+                }
+                catch(Exception e) 
+            	{ 
+                	System.out.println("Error processing line!"); 
+                	continue; 
+            	};
+            }
+        }
+    }
+
+    public void save(String path) throws Exception
+    {
+    	File fout = new File(path);
+		FileOutputStream fos = new FileOutputStream(fout);
+	 
+		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos));
+	 
+		embeddings.forEach((key, value) -> {
+			try {
+				writer.write(key + " ");
+			} catch (Exception e1) {
+				// TODO Auto-generated catch block
+				e1.printStackTrace();
+			}
+            for(int i = 0; i < value.length; i++)
+            {
+                try {
+					writer.write(String.valueOf(value[i]) + " ");
+				} catch (Exception e) {
+					// TODO Auto-generated catch block
+					e.printStackTrace();
+				}
+            }
+            try {
+				writer.newLine();
+			} catch (Exception e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+		});
+		writer.close();
+    }
+
+    public double similarity(String word1, String word2)
+    {
+        if (word1.compareTo(word2) == 0) return 1;
+        if (embeddings.containsKey(word1) && embeddings.containsKey(word2))
+        {
+            try {
+				return VectorOperations.cosine(embeddings.get(word1), embeddings.get(word2));
+			} catch (Exception e) {
+				// TODO Auto-generated catch block
+				return -2;
+			}
+        }
+        else return -2;
+    }
+
+    public double[] getEmbedding(String word)
+    {
+        if (embeddings.containsKey(word)) return embeddings.get(word);
+        else return null;
+    }
+
+    public List<Pair<String, Double>> getMostSimilar(String word, int numMostSimilar)
+    {
+        List<Pair<String, Double>> mostSimilar = new ArrayList<Pair<String, Double>>();
+        if (embeddings.containsKey(word))
+        {
+            embeddings.forEach((key, val) -> {
+            	if (key.trim() != word)
+                {
+                    double sim;
+					try {
+						sim = VectorOperations.cosine(embeddings.get(word), val);
+					} catch (Exception e) {
+						// TODO Auto-generated catch block
+						sim = -2;
+					}
+                    if (mostSimilar.size() < numMostSimilar)
+                    {
+                        mostSimilar.add(new Pair<String, Double>(key, sim));
+                        mostSimilar.sort((x,y) -> x.getValue1() > y.getValue1() ? -1 : (x.getValue1() < y.getValue1() ? 1 : 0));
+                    }
+                    else if (sim > mostSimilar.get(mostSimilar.size() - 1).getValue1())
+                    {
+                        mostSimilar.set(mostSimilar.size() - 1, new Pair<String, Double>(key, sim));
+                        mostSimilar.sort((x,y) -> x.getValue1() > y.getValue1() ? -1 : (x.getValue1() < y.getValue1() ? 1 : 0));
+                    }
+                }
+            });
+            
+            return mostSimilar;
+        }
+        else return null;
+    }
+}

source/src/edu/uma/nlp/graphseg/utils/ApplicationConfiguration.java

@@ -0,0 +1,49 @@
+package edu.uma.nlp.graphseg.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class ApplicationConfiguration {
+	
+	public static ApplicationConfiguration config = new ApplicationConfiguration();
+		
+	private Properties prop;
+	
+	public ApplicationConfiguration() 
+	{
+		prop = new Properties();
+		InputStream inStream = getClass().getClassLoader().getResourceAsStream("config.properties");
+		
+		if (inStream != null)
+		{
+			try 
+			{
+				prop.load(inStream);
+				
+			} 
+			catch (IOException e) {
+				e.printStackTrace();
+			}
+			finally
+			{
+				try 
+				{
+					inStream.close();
+				}
+				catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
+	
+	public String getValue(String key)
+	{
+		if (prop != null)
+		{
+			return prop.getProperty(key);
+		}
+		else return null;
+	}
+}

source/src/edu/uma/nlp/graphseg/utils/IOHelper.java

@@ -0,0 +1,385 @@
+package edu.uma.nlp.graphseg.utils;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class IOHelper {
+	public static List<String> getAllLines(String path)
+	{
+		try {
+			return FileUtils.readLines(new File(path));
+		} catch (IOException e) {
+			System.out.println("File not found or error reading the file: " + path);
+			System.out.println(e.getMessage());
+			return null;
+		}
+	}
+	
+	public static List<String> getAllLinesWithoutEmpty(String path)
+	{
+		try {
+			List<String> alllines = Files.readAllLines(Paths.get(path));
+			List<String> noEmpty = new ArrayList<String>();
+			
+			for(int i = 0; i < alllines.size(); i++)
+			{
+				if (!StringUtils.isEmpty(alllines.get(i).trim())) 
+				{
+					noEmpty.add(alllines.get(i));
+				}
+			}
+			
+			return noEmpty;
+			
+		} catch (IOException e) {
+			System.out.println("File not found or error reading the file: " + path);
+			return null;
+		}
+	}
+	
+	
+	public static void writeLines(List<String> lines, String path)
+	{
+		StringBuilder builder = new StringBuilder();
+		for(int i = 0; i < lines.size(); i++)
+		{
+			builder.append(lines.get(i) + "\n");
+		}
+		
+		try {
+			FileUtils.writeStringToFile(new File(path), builder.toString());
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+	
+	public static void writeCounts(Map<String, Integer> dictionary, String path, Boolean ordered)
+	{
+		writeCounts(dictionary.entrySet().stream().collect(Collectors.toList()), path, ordered);
+	}
+	
+	public static void writeCounts(List<Map.Entry<String, Integer>> entries, String path, Boolean ordered)
+	{
+		try {
+			File fout = new File(path);
+			FileOutputStream fos;
+			fos = new FileOutputStream(fout);
+			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos, "UTF-8"));
+			
+			if (ordered) entries.sort((i1, i2) -> i1.getValue() > i2.getValue() ? -1 : (i2.getValue() > i1.getValue() ? 1 : 0));
+			
+			for(int i = 0; i < entries.size(); i++)
+			{
+				bw.write(entries.get(i).getKey() + " " + entries.get(i).getValue() + "\n");
+			}
+			
+			bw.close();
+			
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	
+	}
+	
+	public static void writeScores(Map<String, Double> dictionary, String path, Boolean orderedDescending, Map<String, Integer> additionalData, Boolean mweUnderscore)
+	{
+		writeScores(dictionary.entrySet().stream().collect(Collectors.toList()), path, orderedDescending, additionalData, mweUnderscore);
+	}
+	
+	public static void writeScores(List<Map.Entry<String, Double>> entries, String path, Boolean orderedDescending, Map<String, Integer> additionalData, Boolean mweUnderscore)
+	{
+		try {
+			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(path)), "UTF-8"));
+			
+			entries.sort((i1, i2) -> i1.getValue() > i2.getValue() ? (orderedDescending ? -1 : 1) : (i2.getValue() > i1.getValue() ? (orderedDescending ? 1 : -1) : 0));
+			
+			for(int i = 0; i < entries.size(); i++)
+			{
+				String line = "";
+				if (mweUnderscore)
+				{
+					String[] split = entries.get(i).getKey().split("\\s+");
+					StringBuilder singlePhrase = new StringBuilder();
+					for(int j = 0; j < split.length; j++)
+					{
+						singlePhrase.append(split[j]);
+						if (j < split.length - 1) singlePhrase.append("_");
+					}
+					line += singlePhrase.toString() + " ";
+				}
+				else line = entries.get(i).getKey() + " ";
+				
+				line += String.valueOf(entries.get(i).getValue());
+				
+				if (additionalData != null)
+				{
+					line += " " + String.valueOf(additionalData.get(entries.get(i).getKey()));
+				}
+				bw.write(line.trim() + "\n");
+			}
+			
+			bw.close();
+			
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	
+	}
+	
+	public static HashMap<String, Integer> loadCounts(String path)
+	{
+		HashMap<String, Integer> dict = new HashMap<String, Integer>();
+		List<String> lines = getAllLines(path);
+		
+		for(int i = 0; i < lines.size(); i++)
+		{
+			//if (i % 100 == 0) System.out.println("Loading counts: " + String.valueOf(i+1) + "/" + String.valueOf(lines.size()));
+			String split[] = lines.get(i).split("\\s+");
+			if (!dict.containsKey(split[0]))
+			{
+				dict.put(split[0], Integer.parseInt(split[1]));
+			}
+		}
+		return dict;
+	}
+	
+	public static HashMap<String, Integer> loadCounts(InputStream stream) throws UnsupportedEncodingException, IOException
+	{
+		HashMap<String, Integer> dict = new HashMap<String, Integer>();
+		try(BufferedReader br = new BufferedReader(new InputStreamReader(stream, "UTF-8"))) {
+		    for(String line; (line = br.readLine()) != null; ) {
+		    	if (StringUtils.isNotEmpty(line.trim()))
+		    	{
+		    		//if (i % 100 == 0) System.out.println("Loading counts: " + String.valueOf(i+1) + "/" + String.valueOf(lines.size()));
+					String split[] = line.split("\\s+");
+					if (!dict.containsKey(split[0]))
+					{
+						dict.put(split[0], Integer.parseInt(split[1]));
+					}
+		    	}
+		    }
+		}
+		return dict;
+	}
+	
+	public static HashMap<String, Double> loadScores(String path)
+	{
+		HashMap<String, Double> dict = new HashMap<String, Double>();
+		List<String> lines = getAllLines(path);
+		
+		for(int i = 0; i < lines.size(); i++)
+		{
+			//if (i % 100 == 0) System.out.println("Loading counts: " + String.valueOf(i+1) + "/" + String.valueOf(lines.size()));
+			String split[] = lines.get(i).split("\\s+");
+			if (!dict.containsKey(split[0]))
+			{
+				dict.put(split[0], Double.parseDouble(split[1]));
+			}
+		}
+		return dict;
+	}
+	
+	public static void peekTopLines(String inputpath, String outputPath, int numLines)
+	{	
+		List<String> lines = new ArrayList<String>();
+		try(BufferedReader br = new BufferedReader(new FileReader(inputpath))) {
+		    for(int i = 0; i < numLines; i++) {
+		    	lines.add(br.readLine());
+	    	}
+		    IOHelper.writeLines(lines, outputPath);
+		    // line is not visible here.
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+	
+	public static Map<String, Double> loadScoresLineByLine(String path)
+	{
+		Map<String, Double> dict = Collections.synchronizedMap(new HashMap<String, Double>());
+		
+		try(BufferedReader br = new BufferedReader(new FileReader(path))) {
+		    for(String line; (line = br.readLine()) != null; ) {
+		    	if (StringUtils.isNotEmpty(line.trim()))
+		    	{
+			    	String split[] = line.split("\\s+");
+					if (!dict.containsKey(split[0]))
+					{
+						dict.put(split[0], Double.parseDouble(split[1]));
+					}
+		    	}
+		    }
+		    // line is not visible here.
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+		return dict;
+	}
+	
+	public static Map<String, Integer> loadRanks(String path)
+	{
+		Map<String, Integer> dict = Collections.synchronizedMap(new HashMap<String, Integer>());
+
+		try(BufferedReader br = new BufferedReader(new FileReader(path))) {
+			int counter = 0;
+			for(String line; (line = br.readLine()) != null; ) {
+		    	if (StringUtils.isNotEmpty(line.trim()))
+		    	{
+		    		counter++;
+			    	String split[] = line.split("\\s+");
+					if (!dict.containsKey(split[0]))
+					{
+						dict.put(split[0], counter);
+					}
+		    	}
+		    }
+		    // line is not visible here.
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+		return dict;
+	}
+	
+	public static Map<String, Double> loadScoresLineByLine(String path, double treshold, Boolean sorted)
+	{
+		Map<String, Double> dict = Collections.synchronizedMap(new HashMap<String, Double>());
+		
+		try(BufferedReader br = new BufferedReader(new FileReader(path))) {
+		    for(String line; (line = br.readLine()) != null; ) {
+		    	if (StringUtils.isNotEmpty(line.trim()))
+		    	{
+			    	String split[] = line.split("\\s+");
+					if (!dict.containsKey(split[0]))
+					{
+						Double score = Double.parseDouble(split[1]);
+						if (score >= treshold) dict.put(split[0], score);
+						else if (sorted) 
+						{
+							return dict;
+						}
+					}
+		    	}
+		    }
+		    // line is not visible here.
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+		return dict;
+	}
+	
+	public static Map<String, Double> loadScoresLineByLine(String path, int topN)
+	{
+		Map<String, Double> dict = Collections.synchronizedMap(new HashMap<String, Double>());
+		
+		try(BufferedReader br = new BufferedReader(new FileReader(path))) {
+		    for(int i = 0; i < topN; i++) {
+		    	String line = br.readLine();
+		    	if (StringUtils.isNotEmpty(line.trim()))
+		    	{
+			    	String split[] = line.split("\\s+");
+					if (!dict.containsKey(split[0]))
+					{
+						Double score = Double.parseDouble(split[1]);
+						dict.put(split[0], score);
+					}
+		    	}
+		    }
+		    // line is not visible here.
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+		return dict;
+	}
+	
+	public static HashMap<String, String> loadMappings(String path)
+	{
+		HashMap<String, String> dict = new HashMap<String, String>();
+		List<String> lines = getAllLines(path);
+		
+		for(int i = 0; i < lines.size(); i++)
+		{
+			//if (i % 100 == 0) System.out.println("Loading mappings: " + String.valueOf(i+1) + "/" + String.valueOf(lines.size()));
+			String split[] = lines.get(i).split("\\s+");
+			if (!dict.containsKey(split[0]))
+			{
+				dict.put(split[0], split[1]);
+			}
+		}
+		return dict;
+	}
+	
+	public static HashMap<String, List<String>> loadMultiMappings(String path)
+	{
+		HashMap<String, List<String>> dict = new HashMap<String, List<String>>();
+		List<String> lines = getAllLines(path);
+		
+		for(int i = 0; i < lines.size(); i++)
+		{
+			//if (i % 100 == 0) System.out.println("Loading mappings: " + String.valueOf(i+1) + "/" + String.valueOf(lines.size()));
+			String split[] = lines.get(i).split("\\s+");
+			if (!dict.containsKey(split[0]))
+			{
+				dict.put(split[0], new ArrayList<String>());
+			}
+			dict.get(split[0]).add(split[1]);
+		}
+		return dict;
+	}
+	
+	public static void writeStringToFile(String content, String path) throws IOException
+	{
+		FileUtils.writeStringToFile(new File(path), content, "UTF-8");
+	}
+}

source/src/edu/uma/nlp/graphseg/utils/MemoryStorage.java

@@ -0,0 +1,26 @@
+package edu.uma.nlp.graphseg.utils;
+
+import edu.uma.nlp.graphseg.semantics.InformationContent;
+import edu.uma.nlp.graphseg.semantics.WordVectorSpace;
+
+public class MemoryStorage {
+
+    private static WordVectorSpace wordVectorSpace;
+	
+	public static WordVectorSpace getWordVectorSpace() {
+		return wordVectorSpace;
+	}
+	public static void setWordVectorSpace(WordVectorSpace wordVectorSpace) {
+		MemoryStorage.wordVectorSpace = wordVectorSpace;
+	}
+	
+	private static InformationContent informationContent;
+	
+	public static InformationContent getInformationContent() {
+		return informationContent;
+	}
+	public static void setInformationContent(InformationContent informationContent) {
+		MemoryStorage.informationContent = informationContent;
+	}
+	
+}

source/src/edu/uma/nlp/graphseg/utils/VectorOperations.java

@@ -0,0 +1,45 @@
+package edu.uma.nlp.graphseg.utils;
+
+public class VectorOperations {
+	public static double cosine(double[] vector, double[] otherVector) throws Exception
+    {
+        if (vector.length != otherVector.length) 
+    	{
+        	throw new UnsupportedOperationException("Vectors are of different length");
+    	}
+        
+        double dp = 0;
+        double sum1 = 0;
+        double sum2 = 0;
+        
+        for (int i = 0; i < vector.length; i++)
+        {
+            dp += vector[i] * otherVector[i];
+            sum1 += vector[i] * vector[i];
+            sum2 += otherVector[i] * otherVector[i];
+        }
+
+        return dp / (Math.sqrt(sum1) * Math.sqrt(sum2));
+    }
+	
+    public static void multiply(double[] vector, double factor)
+    {
+        for (int i = 0; i < vector.length; i++) vector[i] *= factor;
+    }
+
+    public double[] sumVectors(double[] vector, double[] otherVector)
+    {
+        if (vector.length != otherVector.length) throw new UnsupportedOperationException("Vectors are of different length");
+        
+        double[] result = new double[vector.length];
+        for (int i = 0; i < vector.length; i++) result[i] = vector[i] += otherVector[i];
+        
+        return result;
+    }
+    
+    public static void addVector(double[] vector, double[] otherVector)
+    {
+        if (vector.length != otherVector.length) throw new UnsupportedOperationException("Vectors are of different length");
+        for (int i = 0; i < vector.length; i++) vector[i] += otherVector[i];
+    }
+}