Lec05: 条件抽出, Apply関数

作業ディレクトリの設定

setwd("/cloud/project")

参考サイト

• Rのデータ型とデータ構造の基本
• cleanNLPパッケージ

getWordLst

wdLst.BBC <- getWordFreq("G7/BBC.txt")
head(wdLst.BBC)

wordLst
    the      in      of      to       a ukraine 
     24      15      13      10       9       7 

apply関数

単語頻度表の一部を抽出

(tmp<-wdLst.BBC[1:5])

wordLst
the  in  of  to   a 
 24  15  13  10   9 

lapply関数

• paste関数

paste(tmp, "@BBC")

[1] "24 @BBC" "15 @BBC" "13 @BBC" "10 @BBC" "9 @BBC" 

lapply

lapply(tmp, paste, "@BBC")

$the
[1] "24 @BBC"

$`in`
[1] "15 @BBC"

$of
[1] "13 @BBC"

$to
[1] "10 @BBC"

$a
[1] "9 @BBC"

sapply:名前の属性付き

sapply(tmp, paste, "@BBC")

      the        in        of        to         a 
"24 @BBC" "15 @BBC" "13 @BBC" "10 @BBC"  "9 @BBC" 

apply関数

相対頻度

relativeFreq <- wdLst.BBC/sum(wdLst.BBC)*100
head(relativeFreq)

wordLst
     the       in       of       to        a  ukraine 
5.839416 3.649635 3.163017 2.433090 2.189781 1.703163 

粗頻度と相対頻度をcbindで結合

tmpMtx <- cbind(wdLst.BBC, relativeFreq)
colnames(tmpMtx)<- c("Freq", "RelativFreq")
head(tmpMtx)

        Freq RelativFreq
the       24    5.839416
in        15    3.649635
of        13    3.163017
to        10    2.433090
a          9    2.189781
ukraine    7    1.703163

Apply on rows

res <- apply(tmpMtx,1,sum)
head(res)

      the        in        of        to         a   ukraine 
29.839416 18.649635 16.163017 12.433090 11.189781  8.703163 

Apply on columns

apply(tmpMtx,2,sum)

       Freq RelativFreq 
        411         100 

Apply on elements

res <- apply(tmpMtx,c(1,2), function(x) x*10)
head(res)

        Freq RelativFreq
the      240    58.39416
in       150    36.49635
of       130    31.63017
to       100    24.33090
a         90    21.89781
ukraine   70    17.03163

ライブラリの読み込み

library(cleanNLP)

テキスト処理

言語モデルの設定model_name （デフォルト値=英語）

• UDPipe
• Universal Dependencies

cnlp_init_udpipe()

全文結合

• UDPipe</a

readLines("ou_msg/ou_msg_en.txt") %>% keep(~ . != "") %>% paste(., collapse = " ") -> all_txt
substr(all_txt, start=1, stop=100)

[1] "Greetings, everyone. Today, I was appointed as the President of Osaka University by the Minister of "

指定ディレクトリのファイル一覧を取得(相対パス)

dirName <-"G7"
files<- list.files(dirName)
files

[1] "Aljazeera.txt"  "BBC.txt"        "JapanTimes.txt"

filesDir <- unlist(lapply(dirName, paste, files, sep = "/"))
filesDir

[1] "G7/Aljazeera.txt"  "G7/BBC.txt"        "G7/JapanTimes.txt"

join_Lines <- function(fnaame) {
  txt <- readLines(fnaame)
  txt<- txt[txt != ""]
  paste(txt, collapse = " ")
}

lapply(filesDir, join_Lines) %>% unlist -> G7_txtset

• cnlp_annotate関数

res <- cnlp_annotate(input = G7_txtset)

行列のサイズ: dim関数

dim(res$token)

[1] 1646   11

条件抽出: 句読点,記号, 数字を除外

res <- res$token[!res$token$upos %in% c("PUNCT","SYM","NUM"),]
dim(res)

[1] 1425   11

View関数

View(res)

部分抽出（列名指定）

head(res$lemma)

[1] "top"      "diplomat" "from"     "the"      "group"    "of"      

部分抽出（列インデックス指定）

head(res[,6])

部分抽出（列名による条件抽出）

colnames(res) #列名リスト

 [1] "doc_id"        "sid"           "tid"           "token"         "token_with_ws" "lemma"        
 [7] "upos"          "xpos"          "feats"         "tid_source"    "relation"     

head(res[,colnames(res)=="lemma"])

部分抽出（要素による条件抽出）

res[res$lemma=="'s",]

文書行列(Term-Document Matrix)

docMtx <- table(res$lemma, res$doc_id)
head(docMtx)

           
             1  2  3
  -escalate  0  0  1
  's         7  5  6
  &          0  0  1
  a          7 13 20
  ability    0  1  0
  about      2  2  0

#typeof(docMtx)

列名

colnames(docMtx)

[1] "1" "2" "3"

列名の変更

files %>% strsplit(".txt") %>% unlist -> colnames(docMtx)

変更確認

head(docMtx)

           
            Aljazeera BBC JapanTimes
  -escalate         0   0          1
  's                7   5          6
  &                 0   0          1
  a                 7  13         20
  ability           0   1          0
  about             2   2          0

文書行列のサイズ

dim(docMtx)

[1] 522   3

Head抽出

head(docMtx)

           
            Aljazeera BBC JapanTimes
  -escalate         0   0          1
  's                7   5          6
  &                 0   0          1
  a                 7  13         20
  ability           0   1          0
  about             2   2          0

View

View(docMtx)

#as.data.frame(docMtx)

Sorting the rows of docMtx based on the word frequencies of JapanTimes

tmpMtx <- docMtx[order(docMtx[,3],decreasing=TRUE),]
head(tmpMtx)

     
      Aljazeera BBC JapanTimes
  the        21  24         46
  in         15  15         22
  a           7  13         20
  on          3   5         20
  to         11  10         19
  and        13   4         14

Calculating overall word frequencies with rowSums()

wFreqs <- rowSums(docMtx)
head(wFreqs)

-escalate        's         &         a   ability     about 
        1        18         1        40         1         4 

Sorting the docMtx by overall word frequencies

docMtx <- docMtx[order(rowSums(docMtx),decreasing=TRUE),]
head(docMtx)

     
      Aljazeera BBC JapanTimes
  the        21  24         46
  in         15  15         22
  a           7  13         20
  to         11  10         19
  be         10  11         13
  of         14  13          7

Calculating each tokens with colSums()

colSums(docMtx)

 Aljazeera        BBC JapanTimes 
       390        409        626 

Cf. Calculating each types with apply()

apply(docMtx, 2, sum)

 Aljazeera        BBC JapanTimes 
       390        409        626 

Calculating each types with colSums()

#apply(docMtx, 2, function(x) x[x>0] )
apply(docMtx, 2, function(x) length(x[x>0]) )

 Aljazeera        BBC JapanTimes 
       207        229        281 

Calculating each TTR

apply(docMtx, 2, function(x) length(x[x>0])/sum(x)*100 )

 Aljazeera        BBC JapanTimes 
  53.07692   55.99022   44.88818