TermDocumentMatrix.cs

Go to the documentation of this file.

// TraceLab Component Library
// Copyright © 2012-2013 SEMERU
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using TraceLab.Components.DevelopmentKit.Properties;
using TraceLabSDK;
using TraceLabSDK.Types;

namespace TraceLab.Components.DevelopmentKit
{
    public class TermDocumentMatrix
    {
        #region Private members
        private double[][] _matrix;
        private List<string> _termIndex;
        private List<string> _docIndex;
        private Dictionary<string, int> _termIndexLookup;
        private Dictionary<string, int> _docIndexLookup;
        #endregion

        #region Public accessors

        public double this[int docindex, int termindex]
        {
            get
            {
                return _matrix[docindex][termindex];
            }
            set
            {
                _matrix[docindex][termindex] = value;
            }
        }

        public double[][] RawMatrix
        {
            get
            {
                return _matrix;
            }
        }

        public List<string> TermMap
        {
            get
            {
                return _termIndex;
            }
        }

        public List<string> DocMap
        {
            get
            {
                return _docIndex;
            }
        }

        public int NumDocs
        {
            get
            {
                return _docIndex.Count;
            }
        }

        public int NumTerms
        {
            get
            {
                return _termIndex.Count;
            }
        }

        #endregion

        #region Constructor

        public TermDocumentMatrix(params TLArtifactsCollection[] artifactsCollections)
        {
            _termIndex = new List<string>();
            _docIndex = new List<string>();
            _termIndexLookup = new Dictionary<string, int>();
            _docIndexLookup = new Dictionary<string, int>();

            // create temporary corpus to build matrix with
            Dictionary<string, Dictionary<string, double>> corpus = new Dictionary<string, Dictionary<string, double>>();
            foreach (TLArtifactsCollection artifacts in artifactsCollections)
            {
                foreach (TLArtifact artifact in artifacts.Values)
                {
                    // update document maps
                    _docIndex.Add(artifact.Id);
                    _docIndexLookup.Add(artifact.Id, _docIndex.Count - 1);
                    corpus.Add(artifact.Id, new Dictionary<string, double>());
                    foreach (string term in artifact.Text.Split())
                    {
                        if (!String.IsNullOrWhiteSpace(term))
                        {
                            // update term maps
                            if (!_termIndexLookup.ContainsKey(term))
                            {
                                _termIndex.Add(term);
                                _termIndexLookup.Add(term, _termIndex.Count - 1);
                            }
                            // update document counts
                            if (corpus[artifact.Id].ContainsKey(term))
                            {
                                corpus[artifact.Id][term]++;
                            }
                            else
                            {
                                corpus[artifact.Id].Add(term, 1);
                            }
                        }
                    }
                }
            }

            // build term-by-document matrix
            _matrix = new double[_docIndex.Count][];
            for (int i = 0; i < _docIndex.Count; i++)
            {
                _matrix[i] = new double[_termIndex.Count];
                for (int j = 0; j < _termIndex.Count; j++)
                {
                    corpus[_docIndex[i]].TryGetValue(_termIndex[j], out _matrix[i][j]);
                }
            }
        }

        public TermDocumentMatrix(int docs, int terms)
        {
            _matrix = new double[docs][];
            _docIndex = new List<string>();
            _docIndexLookup = new Dictionary<string, int>();
            for (int i = 0; i < docs; i++)
            {
                _matrix[i] = new double[terms];
                _docIndex.Add("d" + i);
                _docIndexLookup.Add("d" + i, i);
            }
            _termIndex = new List<string>();
            _termIndexLookup = new Dictionary<string, int>();
            for (int i = 0; i < terms; i++)
            {
                _termIndex.Add("t" + i);
                _termIndexLookup.Add("t" + i, i);
            }
        }

        private TermDocumentMatrix() { }

        public TermDocumentMatrix(TermDocumentMatrix matrix)
        {
            _matrix = new double[matrix.NumDocs][];
            for (int i = 0; i < matrix.NumDocs; i++)
            {
                _matrix[i] = new double[matrix.NumTerms];
                for (int j = 0; j < matrix.NumTerms; j++)
                {
                    _matrix[i][j] = matrix[i,j];
                }
            }
            _docIndex = new List<string>(matrix._docIndex);
            _docIndexLookup = new Dictionary<string, int>(matrix._docIndexLookup);
            _termIndex = new List<string>(matrix._termIndex);
            _termIndexLookup = new Dictionary<string, int>(matrix._termIndexLookup);
        }

        #endregion

        #region Getters

        public double[] GetDocument(int index)
        {
            return _matrix[index];
        }

        public double[] GetDocument(string artifactID)
        {
            return GetDocument(_docIndexLookup[artifactID]);
        }

        public double GetValue(int doc, int term)
        {
            return _matrix[doc][term];
        }

        public double GetValue(string artifactID, string term)
        {
            return GetValue(_docIndexLookup[artifactID], _termIndexLookup[term]);
        }

        public int GetTermIndex(string term)
        {
            return _termIndexLookup[term];
        }

        public string GetTermName(int index)
        {
            return _termIndex[index];
        }

        public int GetDocumentIndex(string artifactID)
        {
            return _docIndexLookup[artifactID];
        }

        public string GetDocumentName(int index)
        {
            return _docIndex[index];
        }

        #endregion

        #region Setters

        public void SetDocument(int index, double[] doc)
        {
            if (doc.Length != _matrix[index].Length)
                throw new ArgumentException("The array sizes do not match.");
            _matrix[index] = doc;
        }

        public void SetDocument(string artifactID, double[] doc)
        {
            SetDocument(_docIndexLookup[artifactID], doc);
        }

        public void SetValue(int doc, int term, double value)
        {
            _matrix[doc][term] = value;
        }

        public void SetValue(string artifactID, string term, double value)
        {
            SetValue(_docIndexLookup[artifactID], _termIndexLookup[term], value);
        }

        public void SetMatrix(double[][] matrix)
        {
            if (matrix.GetLength(0) != _matrix.GetLength(0))
                throw new ArgumentException("The matrix has the wrong number of rows.");
            for (int i = 0; i < _matrix.GetLength(0); i++)
            {
                if (matrix[i].Length != _matrix[i].Length)
                    throw new ArgumentException("The matrix has the wrong number of columns in row " + i + ".");
            }
            _matrix = matrix;
        }

        #endregion

        #region Queries

        public bool ContainsDocument(string artifactID)
        {
            return _docIndexLookup.ContainsKey(artifactID);
        }

        public bool ContainsTerm(string term)
        {
            return _termIndexLookup.ContainsKey(term);
        }

        public TLArtifactsCollection ToTLArtifactsCollection()
        {
            TLArtifactsCollection artifacts = new TLArtifactsCollection();
            for (int i = 0; i < NumDocs; i++)
            {
                List<string> text = new List<string>();
                for (int j = 0; j < NumTerms; j++)
                {
                    if (_matrix[i][j] > 0.0 && Math.Abs(_matrix[i][j] - Math.Round(_matrix[i][j])) < Settings.Default.DoubleTolerance)
                    {
                        int kmax = Convert.ToInt32(Math.Round(_matrix[i][j]));
                        for (int k = 0; k < kmax; k++)
                        {
                            text.Add(GetTermName(j));
                        }
                    }
                }
                artifacts.Add(new TLArtifact(GetDocumentName(i), String.Join(" ", text)));
            }
            return artifacts;
        }

        #endregion

        #region Static utilities

        public static List<TermDocumentMatrix> Equalize(TermDocumentMatrix matrix1, TermDocumentMatrix matrix2)
        {
            // initialize matrices
            List<TermDocumentMatrix> matrices = new List<TermDocumentMatrix>();
            // matrix 1
            matrices.Add(new TermDocumentMatrix());
            matrices[0]._matrix = new double[matrix1.NumDocs][];
            matrices[0]._docIndex = new List<string>(matrix1._docIndex);
            matrices[0]._docIndexLookup = new Dictionary<string,int>(matrix1._docIndexLookup);
            // matrix 2
            matrices.Add(new TermDocumentMatrix());
            matrices[1]._matrix = new double[matrix2.NumDocs][];
            matrices[1]._docIndex = new List<string>(matrix2._docIndex);
            matrices[1]._docIndexLookup = new Dictionary<string,int>(matrix2._docIndexLookup);
            // compute term set
            List<string> termIndex = new List<string>();
            Dictionary<string, int> termIndexLookup = new Dictionary<string, int>();
            Dictionary<string, int> leftovers = new Dictionary<string,int>(matrix2._termIndexLookup);
            // get all terms in first matrix
            foreach (string term in matrix1._termIndex)
            {
                termIndex.Add(term);
                termIndexLookup.Add(term, termIndex.Count - 1);
                // remove duplicate terms
                if (matrix2._termIndexLookup.ContainsKey(term))
                {
                    leftovers.Remove(term);
                }
            }
            // add leftovers
            foreach (string term in leftovers.Keys)
            {
                termIndex.Add(term);
                termIndexLookup.Add(term, termIndex.Count - 1);
            }
            // create new term distributions for each document
            // matrix 1
            matrices[0]._termIndex = new List<string>(termIndex);
            matrices[0]._termIndexLookup = new Dictionary<string,int>(termIndexLookup);
            for (int i = 0; i < matrices[0].NumDocs; i++)
            {
                matrices[0]._matrix[i] = new double[termIndex.Count];
                // fill in original values
                for (int j = 0; j < matrix1.NumTerms; j++)
                {
                    matrices[0][i, j] = matrix1[i, j];
                }
                // fill in missing terms
                for (int j = matrix1.NumTerms; j < termIndex.Count; j++)
                {
                    matrices[0][i, j] = 0.0;
                }
            }
            // matrix 2
            matrices[1]._termIndex = new List<string>(termIndex);
            matrices[1]._termIndexLookup = new Dictionary<string,int>(termIndexLookup);
            for (int i = 0; i < matrices[1].NumDocs; i++)
            {
                matrices[1]._matrix[i] = new double[termIndex.Count];
                // fill in values
                for (int j = 0; j < termIndex.Count; j++)
                {
                    if (matrix2.ContainsTerm(termIndex[j]))
                    {
                        matrices[1][i, j] = matrix2.GetValue(matrix2.GetDocumentName(i), termIndex[j]);
                    }
                    else
                    {
                        matrices[1][i, j] = 0.0;
                    }
                }
            }
            // return
            return matrices;
        }

        public static TermDocumentMatrix EqualizeDocuments(TermDocumentMatrix matrix1, int document1, TermDocumentMatrix matrix2, int document2)
        {
            // initialize new TermDocumentMatrix
            TermDocumentMatrix newmatrix = new TermDocumentMatrix();
            newmatrix._matrix = new double[2][];
            newmatrix._termIndex = new List<string>();
            newmatrix._termIndexLookup = new Dictionary<string, int>();
            newmatrix._docIndex = new List<string>();
            newmatrix._docIndexLookup = new Dictionary<string, int>();
            newmatrix._docIndex.Add(matrix1.GetDocumentName(document1));
            newmatrix._docIndexLookup.Add(matrix1.GetDocumentName(document1), newmatrix._docIndex.Count - 1);
            newmatrix._docIndex.Add(matrix2.GetDocumentName(document2));
            newmatrix._docIndexLookup.Add(matrix2.GetDocumentName(document2), newmatrix._docIndex.Count - 1);
            List<double> doc1 = new List<double>();
            List<double> doc2 = new List<double>();
            // compute total term set
            Dictionary<string, int> leftovers = new Dictionary<string,int>(matrix2._termIndexLookup);
            foreach (string term in matrix1._termIndex)
            {
                newmatrix._termIndex.Add(term);
                newmatrix._termIndexLookup.Add(term, newmatrix._termIndex.Count - 1);
                doc1.Add(matrix1.GetValue(document1, matrix1.GetTermIndex(term)));
                if (matrix2._termIndexLookup.ContainsKey(term))
                {
                    leftovers.Remove(term);
                    doc2.Add(matrix2.GetValue(document2, matrix2.GetTermIndex(term)));
                }
                else
                {
                    doc2.Add(0.0);
                }
            }
            foreach (string term in leftovers.Keys)
            {
                newmatrix._termIndex.Add(term);
                newmatrix._termIndexLookup.Add(term, newmatrix._termIndex.Count - 1);
                doc1.Add(0.0);
                doc2.Add(matrix2.GetValue(document2, matrix2.GetTermIndex(term)));
            }
            newmatrix._matrix[0] = doc1.ToArray();
            newmatrix._matrix[1] = doc2.ToArray();
            return newmatrix;
        }

        public static TermDocumentMatrix EqualizeDocuments(TermDocumentMatrix matrix1, string artifact1, TermDocumentMatrix matrix2, string artifact2)
        {
            return EqualizeDocuments(matrix1, matrix1.GetDocumentIndex(artifact1), matrix2, matrix2.GetDocumentIndex(artifact2));
        }

        public static TermDocumentMatrix Combine(TermDocumentMatrix matrix1, TermDocumentMatrix matrix2)
        {
            TermDocumentMatrix combined = new TermDocumentMatrix();
            // add documents
            combined._docIndex = new List<string>(matrix1._docIndex);
            combined._docIndexLookup = new Dictionary<string, int>(matrix1._docIndexLookup);
            foreach (string doc in matrix2.DocMap)
            {
                combined._docIndex.Add(doc);
                combined._docIndexLookup.Add(doc, combined.NumDocs - 1);
            }
            // calculate union of terms
            combined._termIndex = new List<string>(matrix1._termIndex);
            combined._termIndexLookup = new Dictionary<string, int>(matrix1._termIndexLookup);
            foreach (string term in matrix2.TermMap)
            {
                if (!combined._termIndexLookup.ContainsKey(term))
                {
                    combined._termIndex.Add(term);
                    combined._termIndexLookup.Add(term, combined.NumTerms - 1);
                }
            }
            // create and populate matrix
            combined._matrix = new double[combined.NumDocs][];
            // matrix1
            for (int i = 0; i < matrix1.NumDocs; i++)
            {
                combined._matrix[i] = new double[combined.NumTerms];
                for (int j = 0; j < combined.NumTerms; j++)
                {
                    if (matrix1.ContainsTerm(combined.TermMap[j]))
                    {
                        combined[i, j] = matrix1[i, matrix1.GetTermIndex(combined.TermMap[j])];
                    }
                    else
                    {
                        combined[i, j] = 0.0;
                    }
                }
            }
            // matrix2
            for (int i = matrix1.NumDocs; i < combined.NumDocs; i++)
            {
                combined._matrix[i] = new double[combined.NumTerms];
                for (int j = 0; j < combined.NumTerms; j++)
                {
                    if (matrix2.ContainsTerm(combined.TermMap[j]))
                    {
                        combined[i, j] = matrix2[i - matrix1.NumDocs, matrix2.GetTermIndex(combined.TermMap[j])];
                    }
                    else
                    {
                        combined[i, j] = 0.0;
                    }
                }
            }
            return combined;
        }

        #endregion

        #region Static I/O

        private static string IODelimeter = " ";

        public static void Save(TermDocumentMatrix matrix, string filename)
        {
            // attempt to create file
            TextWriter tw = new StreamWriter(File.Open(filename, FileMode.Create));
            // print out term list
            foreach (string term in matrix.TermMap)
            {
                tw.Write("{0}{1}", TermDocumentMatrix.IODelimeter, term);
            }
            tw.WriteLine();
            // print out each document
            for (int i = 0; i < matrix.NumDocs; i++)
            {
                tw.Write(matrix.GetDocumentName(i));
                // print out each term
                for (int j = 0; j < matrix.NumTerms; j++)
                {
                    tw.Write("{0}{1}", TermDocumentMatrix.IODelimeter, matrix[i, j]);
                }
                tw.WriteLine();
            }
            // close file
            tw.Flush();
            tw.Close();
        }

        public static TermDocumentMatrix Load(string filename)
        {
            TextReader tr = new StreamReader(File.OpenRead(filename));
            TermDocumentMatrix matrix = new TermDocumentMatrix();
            int lineNum = 1;
            string line = tr.ReadLine();
            string[] delimeter = new string[] { TermDocumentMatrix.IODelimeter, " ", "\t" };
            // read terms
// testing
            List<string> termList = new List<string>(line.Split(delimeter, StringSplitOptions.RemoveEmptyEntries));
            // check for identical terms
            HashSet<string> termSet = new HashSet<string>();
            for (int i = 0; i < termList.Count; i++)
            {
                if (termSet.Contains(termList[i]))
                {
                    termList[i] = termList[i] + new Random().Next();
                    i--;
                }
                else
                {
                    termSet.Add(termList[i]);
                }
            }
// end of testing
            // add to matrix
            matrix._termIndex = termList;
            matrix._termIndexLookup = new Dictionary<string, int>();
            for (int i = 0; i < matrix._termIndex.Count; i++)
            {
                matrix._termIndexLookup.Add(matrix._termIndex[i], i);
            }
            // read documents
            matrix._docIndex = new List<string>();
            matrix._docIndexLookup = new Dictionary<string, int>();
            List<double[]> docs = new List<double[]>();
            while ((line = tr.ReadLine()) != null)
            {
                lineNum++;
                string[] document = line.Split(delimeter, StringSplitOptions.RemoveEmptyEntries);
                if (document.Length != matrix.NumTerms + 1)
                {
                    tr.Close();
                    throw new InvalidDataException("Incorrect data format on line " + lineNum + " in file: " + filename);
                }
                matrix._docIndex.Add(document[0]);
                matrix._docIndexLookup.Add(document[0], matrix._docIndex.Count - 1);
                double[] doc = new double[matrix.NumTerms];
                for (int i = 1; i < document.Length; i++)
                {
                    doc[i - 1] = Convert.ToDouble(document[i]);
                }
                docs.Add(doc);
            }
            // add documents
            matrix._matrix = new double[matrix.NumDocs][];
            for (int i = 0; i < matrix.NumDocs; i++)
            {
                matrix._matrix[i] = new double[matrix.NumTerms];
                for (int j = 0; j < matrix.NumTerms; j++)
                {
                    matrix[i, j] = docs[i][j];
                }
            }
            // cleanup
            tr.Close();
            return matrix;
        }

        public static TermDocumentMatrix LoadTransposed(string filename)
        {
            TermDocumentMatrix original = Load(filename);
            TermDocumentMatrix t = new TermDocumentMatrix();
            t._matrix = new double[original.NumTerms][];
            t._docIndex = new List<string>();
            t._docIndexLookup = new Dictionary<string, int>();
            for (int i = 0; i < original.NumTerms; i++)
            {
                t._matrix[i] = new double[original.NumDocs];
                t._docIndex.Add(original._termIndex[i]);
                t._docIndexLookup.Add(original._termIndex[i], i);
            }
            t._termIndex = new List<string>();
            t._termIndexLookup = new Dictionary<string, int>();
            for (int i = 0; i < original.NumDocs; i++)
            {
                t._termIndex.Add(original._docIndex[i]);
                t._termIndexLookup.Add(original._docIndex[i], i);
            }
            for (int i = 0; i < original.NumTerms; i++)
            {
                for (int j = 0; j < original.NumDocs; j++)
                {
                    t._matrix[i][j] = original._matrix[j][i];
                }
            }
            return t;
        }

        #endregion
    }
}