#matrix with random term frequency (0-10 appearances)
x <- matrix(sample(c(0:10), size=10*15, replace=TRUE), nrow = 10, dimnames = list(c("t1","t2","t3","t4","t5","t6","t7","t8","t9","t10"), c("c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15")))
#manipulation to answer the three questions of similarity
#d1 will have frequent words in common with d2. To represent this, we will replace frequency for terms 1-3 with higher counts across all documents.
for(term in 1:(nrow(x)-7)){
for(doc in 1:ncol(x)){
freq <- sample(c(7:15), size=1, replace=TRUE)
x[term,doc] <- freq
}
}
#to test two docs with no words in common, we will compare d3 with d4, where d4 contains no shared terms with d3
for(term in 1:(nrow(x)-5)){
x[term, 3] <- 0
}
for(term in 6:(nrow(x))){
x[term, 4] <- 0
}
#d5 will have many rare words in common with d6. We will make terms 7-10 "rare" by decreasing their frequency across all documents while increasing their frequency in d5 and d6
for(term in 7:nrow(x)){
for(doc in 1:ncol(x)){
#don't replace term frequency for the null terms of d4 for the previous exercise. We want to increase frequency for d5 and d6
if(doc==5 || doc==6){
freq <- sample(c(10:15), size=1, replace=TRUE)
} else if(doc!=4) {
freq <- sample(c(0:2), size=1, replace=TRUE)
} else { freq=0 }
x[term,doc] <- freq
}
}
x #the final matrix with adjustments for testing c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15
t1 9 14 0 7 10 10 12 14 14 11 14 9 9 9 9
t2 7 8 0 9 9 15 12 14 13 8 15 13 13 12 11
t3 12 10 0 7 15 13 13 11 7 12 11 11 13 15 13
t4 0 4 0 3 5 4 9 2 9 5 6 0 3 0 5
t5 10 3 0 9 6 9 8 2 9 10 9 4 4 9 1
t6 9 2 10 0 9 8 1 2 2 5 2 0 7 2 5
t7 2 0 2 0 13 15 0 1 2 2 1 1 2 0 2
t8 2 1 1 0 11 10 2 0 2 2 2 0 0 1 0
t9 1 2 1 0 15 14 0 1 0 1 0 0 1 0 2
t10 2 0 0 0 14 10 2 1 1 1 0 2 1 0 1corpus <- 15
#compute matrix of term totals if contains term:
y <- matrix(nrow=10, ncol=15) #to be matrix of 0s if term does not appear and 1s if term in doc
for(row in 1:nrow(x)){
for(col in 1:ncol(x)){
if( x[row,col] > 0 ){
y[row,col] <- 1
}
else{
y[row,col] <- 0
}
}
}
dft.sums <- rowSums (y, na.rm = FALSE, dims = 1) #get the dft sum using matrix y
#now, we calculate tf-idf scores for each term and document
for(row in 1:nrow(x)){
for(col in 1:ncol(x)){
tfidf <- 1 + log(x[row,col], 10) + log((corpus/dft.sums[row]), 10)
#check for infinite scores (where 0 occurances of term in document) and replace with 0
if(is.infinite(tfidf)){
x[row,col] <- 0
}
else{
x[row,col] <- tfidf
}
}
}
x #matrix replaced with tf-idf score c1 c2 c3 c4 c5 c6 c7 c8 c9 c10
t1 1.984206 2.176091 0.000000 1.875061 2.029963 2.029963 2.109144 2.176091 2.176091 2.071356
t2 1.875061 1.933053 0.000000 1.984206 1.984206 2.206054 2.109144 2.176091 2.143907 1.933053
t3 2.109144 2.029963 0.000000 1.875061 2.206054 2.143907 2.143907 2.071356 1.875061 2.109144
t4 0.000000 1.736759 0.000000 1.611820 1.833669 1.736759 2.088941 1.435729 2.088941 1.833669
t5 2.029963 1.507084 0.000000 1.984206 1.808114 1.984206 1.933053 1.330993 1.984206 2.029963
t6 2.016390 1.363178 2.062148 0.000000 2.016390 1.965238 1.062148 1.363178 1.363178 1.761118
t7 1.435729 0.000000 1.435729 0.000000 2.248642 2.310790 0.000000 1.134699 1.435729 1.435729
t8 1.477121 1.176091 1.176091 0.000000 2.217484 2.176091 1.477121 0.000000 1.477121 1.477121
t9 1.221849 1.522879 1.221849 0.000000 2.397940 2.367977 0.000000 1.221849 0.000000 1.221849
t10 1.477121 0.000000 0.000000 0.000000 2.322219 2.176091 1.477121 1.176091 1.176091 1.176091
c11 c12 c13 c14 c15
t1 2.176091 1.984206 1.984206 1.984206 1.984206
t2 2.206054 2.143907 2.143907 2.109144 2.071356
t3 2.071356 2.071356 2.143907 2.206054 2.143907
t4 1.912850 0.000000 1.611820 0.000000 1.833669
t5 1.984206 1.632023 1.632023 1.984206 1.029963
t6 1.363178 0.000000 1.907246 1.363178 1.761118
t7 1.134699 1.134699 1.435729 0.000000 1.435729
t8 1.477121 0.000000 0.000000 1.176091 0.000000
t9 0.000000 0.000000 1.221849 0.000000 1.522879
t10 0.000000 1.477121 1.176091 0.000000 1.176091dot.product.matrix <- matrix(nrow=15, ncol=15) #matrix of document by document
# compute dot product of vectors and add to matrix
for(col in 1:(ncol(x))){
c1 <- paste("c",toString(col), sep="")
for(col2 in 1:(ncol(x))){
c2 <- paste("c",toString(col2), sep="")
dot.product <- x[,c1] %*% x[,c2]
dot.product.matrix[col,col2] <- dot.product
}
}
dot.product.matrix #houses dot product of documents [x,y] [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]
[1,] 28.006004 21.629873 9.449556 15.42365 33.00155 33.31639 23.091067 23.076735 25.04954
[2,] 21.629873 23.441057 6.054993 17.51189 27.64923 27.88495 22.745218 21.365064 22.89980
[3,] 9.449556 6.054993 9.189876 0.00000 12.92442 12.82287 3.927535 5.933108 6.60962
[4,] 15.423654 17.511893 0.000000 17.50382 18.42308 18.93994 19.362280 17.237137 19.15418
[5,] 33.001547 27.649225 12.924418 18.42308 44.73839 44.71580 29.369041 29.305252 32.20969
[6,] 33.316390 27.884946 12.822870 18.93994 44.71580 44.81714 29.510345 29.546843 32.50224
[7,] 23.091067 22.745218 3.927535 19.36228 29.36904 29.51034 27.085618 22.377334 26.69772
[8,] 23.076735 21.365064 5.933108 17.23714 29.30525 29.54684 22.377334 23.616021 23.79531
[9,] 25.049541 22.899799 6.609620 19.15418 32.20969 32.50224 26.697719 23.795309 28.63296
[10,] 27.328279 24.768279 8.923146 18.65768 35.44208 35.64913 26.511823 25.323207 28.49192
[11,] 23.410659 23.112578 6.177423 19.36173 29.03510 29.49946 25.144562 22.359557 26.95103
[12,] 19.449771 15.126471 1.629119 15.09665 21.78396 22.27290 18.484254 18.470646 19.40269
[13,] 24.928997 22.533732 7.487254 17.83065 31.65307 31.90984 23.587912 25.015430 25.58379
[14,] 21.058528 19.104921 4.194265 15.97904 22.02385 22.58565 20.383743 17.976246 20.50866
[15,] 23.643962 22.130663 7.553730 16.84966 31.25452 31.36613 22.579297 24.543345 24.49787
[,10] [,11] [,12] [,13] [,14] [,15]
[1,] 27.328279 23.410659 19.449771 24.928997 21.058528 23.64396
[2,] 24.768279 23.112578 15.126471 22.533732 19.104921 22.13066
[3,] 8.923146 6.177423 1.629119 7.487254 4.194265 7.55373
[4,] 18.657678 19.361731 15.096648 17.830648 15.979044 16.84966
[5,] 35.442081 29.035101 21.783955 31.653073 22.023851 31.25452
[6,] 35.649135 29.499455 22.272903 31.909837 22.585650 31.36613
[7,] 26.511823 25.144562 18.484254 23.587912 20.383743 22.57930
[8,] 25.323207 22.359557 18.470646 25.015430 17.976246 24.54334
[9,] 28.491922 26.951030 19.402686 25.583794 20.508665 24.49787
[10,] 30.179637 26.887790 19.302366 27.340887 21.005783 26.49575
[11,] 26.887790 26.816314 17.863713 24.038655 21.072780 22.90916
[12,] 19.302366 17.863713 18.956851 19.004050 16.266675 17.86593
[13,] 27.340887 24.038655 19.004050 26.966215 19.026641 26.27479
[14,] 21.005783 21.072780 16.266675 19.026641 20.430756 17.47981
[15,] 26.495745 22.909158 17.865932 26.274788 17.479812 26.11229euclidian.vector <- c() #will hold euclidian length of documents (in order)
#for each column, multiply each element by itself and sum the total column. Then, sqrt of that total.
for(col in 1:ncol(x)){
sum.euc <- 0
for(row in 1:nrow(x)){
product.euc <- x[row,col]*x[row,col]
sum.euc <- sum.euc + product.euc
}
#add euclidian length to vector
euclidian.vector <- c(euclidian.vector, sqrt(sum.euc))
}
euclidian.vector [1] 5.292070 4.841597 3.031481 4.183756 6.688676 6.694560 5.204385 4.859632 5.350977 5.493600
[11] 5.178447 4.353947 5.192900 4.520039 5.110019cosine.matrix <- matrix(nrow=15, ncol=15) #matrix of document by document
#for each column of our dot product matrix, iterate through the vector of documents to compute the cosine similarity score
for(col in 1:ncol(dot.product.matrix)){
#retrieve euclidian distance for the document on the column
v.d1 <- euclidian.vector[col]
for(row in 1:nrow(dot.product.matrix)){
#for each row document the column document is being compared to, get the row document
v.d2 <- euclidian.vector[row]
#calculate cosine similarity and output to matrix
cosine.sim <- dot.product.matrix[row,col]/(v.d1*v.d2)
cosine.matrix[row,col] <- cosine.sim
}
}
cosine.matrix #this matrix is the final result, housing the cosine similarity score between documents [x,y] [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]
[1,] 1.0000000 0.8441892 0.5890213 0.6966190 0.9323276 0.9403950 0.8383957 0.8973160 0.8845881
[2,] 0.8441892 1.0000000 0.4125439 0.8645261 0.8537961 0.8603183 0.9026765 0.9080552 0.8839139
[3,] 0.5890213 0.4125439 1.0000000 0.0000000 0.6374058 0.6318418 0.2489407 0.4027394 0.4074634
[4,] 0.6966190 0.8645261 0.0000000 1.0000000 0.6583482 0.6762234 0.8892436 0.8478038 0.8555868
[5,] 0.9323276 0.8537961 0.6374058 0.6583482 1.0000000 0.9986166 0.8436848 0.9015751 0.8999395
[6,] 0.9403950 0.8603183 0.6318418 0.6762234 0.9986166 1.0000000 0.8469989 0.9082087 0.9073152
[7,] 0.8383957 0.9026765 0.2489407 0.8892436 0.8436848 0.8469989 1.0000000 0.8847806 0.9586756
[8,] 0.8973160 0.9080552 0.4027394 0.8478038 0.9015751 0.9082087 0.8847806 1.0000000 0.9150712
[9,] 0.8845881 0.8839139 0.4074634 0.8555868 0.8999395 0.9073152 0.9586756 0.9150712 1.0000000
[10,] 0.9400040 0.9312156 0.5358043 0.8117722 0.9645439 0.9693262 0.9272849 0.9485459 0.9692407
[11,] 0.8542570 0.9218500 0.3935075 0.8936722 0.8382694 0.8509272 0.9329861 0.8885058 0.9726189
[12,] 0.8441232 0.7175728 0.1234284 0.8287643 0.7480204 0.7641378 0.8157357 0.8729626 0.8328094
[13,] 0.9071294 0.8962611 0.4756174 0.8207119 0.9113091 0.9178940 0.8727907 0.9912761 0.9207078
[14,] 0.8803598 0.8730004 0.3060968 0.8449717 0.7284686 0.7463943 0.8665075 0.8183770 0.8479340
[15,] 0.8743236 0.8945062 0.4876229 0.7881381 0.9144294 0.9168882 0.8490212 0.9883435 0.8959272
[,10] [,11] [,12] [,13] [,14] [,15]
[1,] 0.9400040 0.8542570 0.8441232 0.9071294 0.8803598 0.8743236
[2,] 0.9312156 0.9218500 0.7175728 0.8962611 0.8730004 0.8945062
[3,] 0.5358043 0.3935075 0.1234284 0.4756174 0.3060968 0.4876229
[4,] 0.8117722 0.8936722 0.8287643 0.8207119 0.8449717 0.7881381
[5,] 0.9645439 0.8382694 0.7480204 0.9113091 0.7284686 0.9144294
[6,] 0.9693262 0.8509272 0.7641378 0.9178940 0.7463943 0.9168882
[7,] 0.9272849 0.9329861 0.8157357 0.8727907 0.8665075 0.8490212
[8,] 0.9485459 0.8885058 0.8729626 0.9912761 0.8183770 0.9883435
[9,] 0.9692407 0.9726189 0.8328094 0.9207078 0.8479340 0.8959272
[10,] 1.0000000 0.9451453 0.8069943 0.9583974 0.8459402 0.9438362
[11,] 0.9451453 1.0000000 0.7922990 0.8939241 0.9002851 0.8657392
[12,] 0.8069943 0.7922990 1.0000000 0.8405297 0.8265583 0.8030084
[13,] 0.9583974 0.8939241 0.8405297 1.0000000 0.8106062 0.9901631
[14,] 0.8459402 0.9002851 0.8265583 0.8106062 1.0000000 0.7567842
[15,] 0.9438362 0.8657392 0.8030084 0.9901631 0.7567842 1.0000000(Comparing d5 and d6 according to term-document matrix set-up)
[1] 0.9986166(Comparing d1 and d2 according to term-document matrix set-up)
[1] 0.8441892(Comparing d3 and d4 according to term-document matrix set-up)
[1] 0