Descriptive analysis

Boxplot

d %>%
  mutate(id_tweet = as.character(id_tweet)) %>%
  select(id_tweet, retweet_count, favorite_count, data_year) %>%
  pivot_longer(cols = -c(data_year, id_tweet), values_to = "values",
               names_to = "variables") %>%
  group_by(data_year, variables) %>%
  # slice(1:1000) %>%
  filter(values > 0) %>%
  mutate(variables = ifelse(variables == "favorite_count",
                            "Likes", "Retweets")) %>%
  ggplot(aes(x = values, y = factor(data_year))) +
  geom_boxplot() +
  xlab("") +
  ylab("Year") +
  scale_x_log10(breaks = log_breaks()) +
  theme_light() +
  facet_wrap(~variables)

Violin plot

The red dot indicates the median value.

d %>%
  mutate(id_tweet = as.character(id_tweet)) %>%
  select(id_tweet, retweet_count, favorite_count, data_year) %>%
  pivot_longer(cols = -c(data_year, id_tweet), values_to = "values",
               names_to = "variables") %>%
  # group_by(data_year, variables) %>%
  # slice(1:1000) %>%
  filter(values > 0) %>%
  mutate(variables = ifelse(variables == "favorite_count",
                            "Likes", "Retweets")) %>%
  ggplot(aes(x = values, y = factor(data_year))) +
  geom_violin() +
  stat_summary(fun=median, geom="point", size=2, color="red") +
  xlab("") +
  ylab("Year") +
  scale_x_log10(breaks = log_breaks()) +
  theme_light() +
  facet_wrap(~variables)

Retweet and like distribution

In this case the y-axis value represents the percentage of tweets instead of the number of tweets.

dd %>%
  mutate(data_year = as_factor(data_year)) %>%
  ggplot(aes(x = values, y = percent_of_tweets, color = data_year)) +
  geom_point() +
  # scale_y_continuous(labels = scales::percent)+
  scale_y_log10(breaks = log_breaks(), labels = scales::percent) +
  scale_x_log10(breaks = trans_breaks("log10", function(x) 10^x),
                labels = trans_format("log10", math_format(10^.x))) +
  scale_color_brewer(palette = "Set1") +
  theme_light() +
  xlab("# rt/likes from a user in collection") +
  ylab("% of tweets") +
  labs(color = "Year") +
  facet_wrap(.~variables)

Common words in word categories

NOTE: The table show the most common words inside categories after first interaction with NEDA content. Table shows top 10 most common words.

read_delim(here::here("data", "liwc", "LIWC2015_English_Flat.dic"),
           delim = "\t", skip = 1, col_names = c("number", "name"),
           n_max = 73) -> categories_name

to_filter <- c("female", "anx", "relig", "risk", "money", "we", "hear", "power", "body", "nonflu",
               "focusfuture", "relativ", "family", "negemo", "focuspresent", "informal", "work",
               "achiev", "social", "they", "focuspast", "health", "tentat", "negate", "ipron",
               "you", "differ", "posemo", "shehe", "affiliation", "see", "ingest", "filler", "friend")

categories_name %>%
  filter(name %in% to_filter) -> selected_categories

read_tsv(here::here("data", "liwc", "LIWC2015_English_Flat.dic"),
         skip = 86, col_names = paste0(c("x"), c("_"), 1:11),
         guess_max = 6000, col_types = "ccccccccccc") %>%
  mutate(x_1 = str_remove(x_1, "\\*")) -> words_dic

find_words <- function(category_number = NA) {
  words_dic %>%
    filter_at(vars(x_2:x_11), any_vars(.==category_number)) %>%
    pull(x_1) %>%
    paste(collapse = "|") -> words_in_we_category

  return(words_in_we_category)
}

final <- function(name, words_inside) {
  neda_liwc_gender %>%
    filter(days_tweet>=0) %>%
    select(text, gender, id_tweet) %>%
    mutate(id_tweet = as.character(id_tweet)) %>%
    mutate(text = str_extract_all(text, words_inside)) %>%
    mutate(text = map_chr(text, ~paste(., collapse = " "))) %>%
    unnest_tokens(word, text) %>%
    count(gender, word, sort = T) %>%
    # top_n(n = 10, wt = n) %>% 
    pivot_wider(names_from = "gender", values_from = "n") %>%
    mutate(total = u + f + m) %>% 
    top_n(n = 10, wt = total) %>% 
    select(word, u, f, m) %>%
    mutate(category = name)
}

neda_liwc <- read_rds("/Volumes/TOSHIBA EXT/neda_nine/data/neda_liwc.rds") %>%
  filter(abs(days_tweet) <= 15) %>%
  select(text, id_tweet, days_tweet)

neda_liwc_gender <- read_rds("/Volumes/TOSHIBA EXT/neda_nine/data/neda_liwc_gender.rds") %>%
  # select(-text, -created_at_tweet) %>%
  filter(abs(days_tweet) <= 15)

neda_liwc_gender %>%
  select(id_tweet, gender, to_filter) %>%
  left_join(neda_liwc) -> neda_liwc_gender

selected_categories %>%
  mutate(words_inside = map_chr(number, ~find_words(.))) %>% 
  # view()
  # slice(5) %>%
  mutate(df = map2(name, words_inside, ~final(.x, .y))) %>%
  select(df) %>%
  unnest(df) %>%
  select(category, word, u, f, m) %>%
  write_csv("change_in_words_category_nineteen.csv") %>%
  gt::gt(groupname_col = "category",
         rowname_col = "word") %>% 
  gt::tab_header(title = "2019")

2019
	u	f	m
we
us	32549	32106	21159
we	28762	28276	17151
our	17956	17957	11911
lets	411	279	244
let's	279	297	213
you
u	226129	223848	157220
you	33214	39520	20653
ye	8566	9711	5870
ya	4972	4584	3750
ily	2624	2352	1597
thou	1805	2221	1069
thy	974	1037	472
thee	143	100	223
y'all	146	158	80
thine	12	3	7
shehe
he	104114	113069	65426
his	15412	18287	10081
she	3795	5271	2115
him	1160	1456	774
oneself	1	4	4
they
they	3480	4626	1946
them	2332	2904	1224
their	2260	2649	1244
ipron
it	60242	65370	39237
this	12200	14821	7863
that	10137	13107	5637
who	4823	5874	2722
what	4748	5614	2890
thing	2337	3112	1464
other	2280	2927	1276
someone	1205	1810	576
these	1356	1602	814
everyone	1070	1176	592
negate
no	28463	32400	18999
np	2231	1930	1302
never	1470	2011	1041
don't	1389	1673	754
arent	798	956	361
can't	486	656	288
without	522	606	285
aint	604	510	323
nah	427	408	270
cant	373	326	247
posemo
ha	53492	61756	34866
ty	9862	10263	6149
ok	9297	10114	6807
love	4725	5808	2803
heal	4655	3755	1567
thank	3925	3389	2375
win	3434	3167	2668
good	2574	2944	1659
ily	2562	2293	1567
care	2218	2193	1076
negemo
ugh	4004	4968	2453
sin	4678	4723	3295
war	4248	4307	2663
low	3748	3879	2196
mad	1906	2185	1406
miss	1366	1831	914
fuck	1278	1804	707
ache	1275	1167	676
cut	909	1179	471
bad	960	1159	550
anx
risk	545	295	288
anxiety	296	544	84
struggl	401	489	143
scare	426	188	166
stress	362	386	120
fear	239	318	145
threat	171	232	103
shame	147	195	67
terror	180	145	86
phobi	169	175	77
social
he	94836	101485	60032
rt	92994	95337	70374
hi	38792	44238	25363
ma	38583	40124	29305
you	33213	39511	20649
us	31429	30828	20405
we	25604	26357	16020
pa	17876	18539	13201
ur	17641	17119	12914
ex	10127	11676	7248
family
ma	40884	42662	30748
pa	18148	18878	13485
kin	8713	9586	6193
son	6478	7143	4209
bro	2600	2810	1865
fam	1804	1650	2064
sis	1759	1958	879
mom	1193	1727	1815
dad	774	848	1018
baby	646	788	415
friend
ally	4970	6564	2601
friend	1596	1964	1011
beau	997	1198	777
guy	965	1131	741
date	1029	880	545
mate	810	887	466
bf	770	676	494
gf	750	758	652
pal	574	675	439
bud	341	460	173
female
ma	40235	41611	30669
her	16182	18697	9880
she	3812	5257	2105
ms	5088	4718	3414
women	3083	3410	1686
girl	1706	2297	1025
sis	1754	1951	877
mom	1193	1726	1813
gal	1170	966	813
lass	769	1027	533
tentat
or	73844	75341	53029
if	13521	14762	8797
some	5372	6509	2978
any	4288	5037	2451
bet	2186	2572	1425
try	2129	2169	1100
may	1670	1824	1021
lot	1375	1536	859
most	1287	1476	831
hope	999	1400	681
differ
or	72309	73629	51716
if	12905	13991	8455
not	7695	9517	4225
jus	5424	6906	3299
but	5244	6799	3179
than	5526	5531	3382
other	2269	2911	1257
nor	2004	2224	1638
really	1566	2120	830
though	1049	1321	601
see
lit	4479	5185	2975
red	5126	5172	3415
see	3204	3943	2094
vid	3126	2644	2474
look	2430	2825	1593
show	2022	2120	1714
pic	1793	2037	1380
watch	1622	1656	1583
view	1405	1053	1152
black	1332	1253	766
hear
ep	11552	12674	7879
ear	8060	9076	5325
ring	4069	4199	2897
sing	2998	3591	2052
say	2476	3022	1309
musi	2967	1441	2290
hear	2336	2787	1412
rap	1752	1850	1711
said	780	1089	437
dj	1032	859	736
body
ear	8558	9685	5594
ass	4102	4532	2859
leg	1908	2241	1266
arch	1826	1480	1232
arm	1701	1627	1217
body	1632	1644	586
hip	1377	1436	844
heart	1258	1432	720
tit	1281	1390	860
itch	1138	1378	603
health
ill	11402	12470	7681
iv	12197	12135	7622
dr	8777	8854	5438
heal	4636	3698	1562
life	2665	2991	2247
live	2115	2103	1685
itch	1063	1317	547
ache	1258	1140	665
fat	741	1062	441
medic	700	661	398
rx	617	509	633
ingest
ate	11501	11229	7988
eat	8408	8712	5822
tea	2734	2846	1982
bar	2376	2106	1835
rum	1735	2253	1161
weigh	1113	678	174
pie	920	1085	661
fat	742	1062	441
food	892	971	474
water	453	648	358
affiliation
hi	41354	47363	26531
us	32378	31839	21059
we	26053	26618	16241
our	17779	17765	11817
kin	8713	9585	6193
ally	4956	6554	2596
love	4755	5831	2826
met	2314	2866	1628
bro	2598	2809	1865
help	2361	2273	1149
achiev
ace	3578	4330	3115
work	3215	3551	2167
win	3435	3167	2669
able	2438	2799	1497
top	2466	2450	1644
try	2129	2169	1100
best	1793	1831	1513
first	1621	1718	1163
gain	1422	1607	898
earn	1474	1227	869
power
up	10918	10570	7436
dr	8045	8071	5167
king	6647	7293	4850
ms	5049	4774	3406
age	4439	4367	2951
war	4291	4346	2655
over	3455	3863	2099
low	2952	3101	1740
win	2925	2746	2402
top	2466	2450	1644
risk
lack	1455	1626	858
stop	1344	1616	741
bad	960	1159	550
loss	937	476	138
lose	660	898	386
safe	713	690	442
risk	545	295	288
wrong	331	420	538
problem	495	482	240
secur	310	334	298
focuspast
ate	12672	12774	8840
was	4831	6196	3109
ran	5760	5577	4391
lit	4481	5185	2975
met	2325	2902	1634
did	1983	2562	1119
been	1742	2303	1106
led	2106	2259	1171
had	1998	2216	1267
got	1766	2136	1084
focuspresent
is	67887	71657	42701
be	33092	37689	23557
am	33948	31952	25092
do	22111	25119	17216
im	17921	19371	12493
go	17300	17998	11828
are	13968	15584	7755
can	9447	10774	6329
eat	8167	8505	5536
have	6736	7955	3807
focusfuture
will	4424	4300	3041
ima	2556	2889	1869
may	1667	1821	1021
going	1305	1628	767
then	1225	1607	702
hope	999	1400	681
gon	1079	1383	709
plan	1061	1354	796
coming	739	775	557
tonight	694	723	601
relativ
in	115582	119052	80824
on	75434	79739	53315
at	64444	70191	42398
go	16193	16685	11000
out	12311	13687	9811
eve	10405	11267	6071
up	11202	10870	7647
day	9487	9507	6372
end	7147	7616	4831
new	6755	5959	5894
work
hr	5992	5748	3714
gre	3630	4155	2808
inc	3397	3674	2367
work	3144	3469	2133
pen	3176	3150	2111
read	2633	2882	1586
test	2569	1611	1957
org	1993	2219	1637
book	1930	2045	2034
school	1139	1452	643
money
fee	2559	3391	1421
owe	2505	2691	1746
bet	2186	2572	1425
lease	2083	2411	1122
irs	1991	2192	1419
rent	1818	2077	1023
free	1631	1343	1445
cent	1622	1394	1088
financ	892	420	1428
owing	928	939	539
relig
sin	4928	5468	3453
rite	1921	1700	943
god	1427	1508	963
hell	845	972	531
christ	759	568	408
angel	611	464	476
amen	534	318	429
pray	410	465	260
alla	375	434	387
zen	379	417	214
informal
o	635909	653746	439491
r	463196	467798	326401
u	244230	246243	167957
b	152071	156943	112124
k	116481	119165	79698
er	103455	109659	71083
ht	71441	64512	47804
ha	41740	46429	27689
yo	37986	44886	23925
da	31554	30827	22879
nonflu
er	104405	110621	71862
um	8574	8857	5752
ah	5720	6251	4027
mm	5902	5694	4191
ugh	4042	4994	2465
oh	3853	3915	2747
zz	2658	1883	2832
hm	1669	1423	967
well	1515	1626	992
uh	980	1078	726
filler
rr	9658	9166	6395
idk	166	211	85
anyway	111	126	38
ykn	38	38	54
whoa	32	34	34
woah	15	34	6
dunno	17	31	5
youknow	23	19	14
blah	21	18	6
imean	2	3	3

neda_liwc <- read_rds(here::here("data", "neda_liwc_twenty.rds")) %>%
  select(-created_at_tweet) %>%
  filter(abs(days_tweet) <= 15)

neda_liwc_gender <- read_rds(here::here("data",
                                        "neda_liwc_gender_twenty.rds")) %>%
  # select(-text, -created_at_tweet) %>%
  filter(abs(days_tweet) <= 15)

selected_categories %>%
  mutate(words_inside = map_chr(number, ~find_words(.))) %>%
  # slice(5) %>%
  mutate(df = map2(name, words_inside, ~final(.x, .y))) %>%
  select(df) %>%
  unnest(df) %>%
  select(category, word, u, f, m) %>%
  write_csv("change_in_words_category_twenty.csv") %>%
  gt::gt(groupname_col = "category",
         rowname_col = "word") %>% 
  gt::tab_header(title = "2020")

2020
	u	f	m
we
us	16900	11586	6610
we	12666	8925	4502
our	7298	4830	2771
let's	189	98	33
lets	104	66	41
you
u	123623	77053	44053
you	17606	10360	6479
ya	7839	3265	1790
ye	5673	3704	1728
ily	1145	736	497
thou	1006	673	437
thy	398	196	139
y'all	126	72	36
thee	72	56	18
thine	3	3	1
shehe
he	52012	36837	19184
his	8496	6196	3223
she	2881	1907	809
him	700	520	329
they
they	2196	1539	958
them	1204	781	469
their	985	764	436
ipron
it	31717	21211	11610
this	7149	5012	2584
that	6001	4297	2393
what	2748	1710	1046
who	2355	1925	1031
thing	1318	843	607
other	1029	739	383
these	732	531	275
someone	696	434	327
everyone	602	423	252
negate
no	18208	11227	6158
np	4312	1861	454
never	944	541	355
don't	923	398	345
dont	503	173	61
can't	419	194	112
nah	328	171	112
arent	285	259	112
aint	281	180	68
without	252	176	110
posemo
ha	64749	27890	12275
ok	5295	3005	1764
ty	5172	3025	1964
love	3387	1916	856
good	1599	963	475
win	1533	1059	629
thank	1421	924	474
heal	1416	1258	391
ily	1114	709	482
care	864	988	315
negemo
sin	2540	1667	1004
ugh	2205	1458	882
war	1869	1695	615
mad	1661	786	406
low	1513	1100	682
nag	1210	559	334
fuck	1033	599	339
miss	1022	675	258
cut	1018	508	250
bad	654	448	251
anx
anxiety	269	108	51
stress	171	82	48
fear	128	98	42
scare	128	79	18
struggl	128	105	79
risk	102	123	84
phobi	119	63	57
panic	111	93	33
threat	73	83	74
avoid	80	73	35
social
rt	53790	32982	18346
he	47015	33315	17138
ma	28889	14388	7750
hi	24226	15541	8300
you	17573	10357	6477
us	16481	11196	6351
pa	12356	7201	3962
we	11504	8373	4263
ur	8110	5661	3328
ya	6621	3097	1867
family
ma	29883	15167	8183
pa	12466	7298	4011
kin	5030	3113	1700
son	3258	2091	1115
bro	992	695	324
sis	829	639	394
dad	699	436	236
fam	691	525	177
mom	648	473	213
baby	601	275	85
friend
ally	3143	2020	1153
friend	820	621	292
guy	813	366	228
date	738	447	192
beau	576	381	123
pal	544	257	121
gf	468	269	121
mate	388	267	115
bf	385	212	95
gang	268	109	31
female
ma	29478	14828	8026
her	8424	5764	3087
she	2868	1902	808
ms	2404	1722	854
girl	1437	699	278
women	1045	842	363
sis	829	636	391
mom	648	473	213
gal	572	317	171
gf	468	269	121
tentat
or	34273	25821	13646
if	6367	4296	2584
some	2916	1717	1196
any	2267	1602	973
may	1940	1007	430
bet	1259	869	386
try	1056	765	350
lot	789	482	295
hope	697	361	251
most	559	476	266
differ
or	33507	25306	13352
if	6077	4096	2467
not	4468	3002	1678
jus	3764	2119	1200
but	3698	2333	1258
than	2216	1529	821
really	1100	616	361
nor	1026	672	388
other	1024	736	383
though	644	412	277
see
lit	2443	1737	935
red	2405	1657	869
see	2105	1372	732
vid	1789	1274	608
look	1505	872	472
pic	1069	586	318
show	1035	559	328
screen	839	284	45
black	832	446	204
watch	786	535	225
hear
ep	5625	3597	2072
ear	3379	2760	1380
say	2859	1220	574
ring	1693	1324	669
sing	1629	1179	651
hear	1164	650	368
rap	1099	582	393
rang	672	275	153
said	560	430	244
dj	483	457	183
body
ear	3445	2860	1455
ass	2130	1511	792
arch	1741	600	256
arm	1070	580	237
hip	982	622	205
leg	913	791	306
heart	724	304	178
itch	695	404	256
hand	680	599	269
tit	570	377	262
health
dr	8858	4416	1617
iv	6177	3689	2147
ill	5989	4104	2367
heal	1392	1253	389
life	1096	583	436
live	890	635	417
itch	655	397	241
rx	439	269	125
ache	371	293	99
wash	340	364	144
ingest
ate	5195	3813	1886
eat	3286	2527	1171
tea	1326	877	402
rum	963	831	251
bar	655	713	319
chees	581	113	23
food	419	234	96
pie	400	316	168
fat	361	210	122
water	328	196	63
sweet	296	191	100
affiliation
hi	25569	16599	8753
us	16832	11520	6571
we	11613	8428	4307
our	7233	4779	2743
kin	5030	3113	1700
love	3404	1922	859
ally	3135	2014	1150
met	1397	689	448
dm	1102	741	285
bro	988	694	324
achiev
ace	1606	1154	764
win	1533	1059	629
work	1243	1277	602
try	1056	766	350
top	1048	753	411
able	962	729	507
first	843	608	271
best	803	562	280
gain	735	481	256
won	684	462	260
power
dr	8658	4176	1552
up	5951	3538	1840
king	3380	2357	1324
ms	2327	1710	827
war	1863	1699	612
age	1520	1225	681
over	1483	1202	615
win	1350	963	455
ses	1281	1006	515
low	1250	927	540
risk
lack	893	559	261
stop	667	479	266
bad	654	448	251
lose	537	302	145
safe	473	322	130
secur	82	113	294
problem	241	125	91
protect	207	174	79
wrong	193	103	77
crisis	76	161	150
focuspast
ate	5765	4295	2050
ran	3921	2212	1723
was	3079	2477	1179
lit	2443	1737	935
met	1398	690	448
did	1397	935	496
got	1143	715	369
had	1010	739	385
been	985	793	401
led	801	821	483
focuspresent
is	32698	23882	13438
be	19714	12776	7003
am	17804	10884	5476
do	16211	9570	5160
im	12663	6828	3722
go	9325	5803	3453
are	6330	4661	2507
can	4856	3664	1729
have	3442	2666	1357
eat	3193	2386	1170
focusfuture
will	2061	1480	821
may	1938	1007	430
ima	1769	917	632
gon	813	477	249
then	734	459	276
going	720	539	304
hope	697	361	251
plan	435	370	174
coming	308	243	112
tonight	304	232	88
relativ
in	58195	39598	21172
on	43402	28605	15010
at	33394	23003	12073
go	8458	5401	3233
up	6119	3683	1901
out	5868	4030	2233
day	5274	2953	1201
eve	5269	3607	2071
end	4081	2669	1315
thin	4056	2430	1570
work
hr	2953	1870	999
pen	1637	1214	733
inc	1471	1274	653
gre	1428	1223	595
work	1212	1254	587
read	1227	1060	466
cv	1114	168	85
org	840	565	288
test	689	611	255
pc	636	390	213
money
fee	1680	817	544
bet	1259	869	386
lease	1131	758	389
owe	1076	837	368
irs	969	704	360
rent	735	624	331
cent	606	479	245
free	442	389	197
owing	381	254	278
econ	319	376	138
relig
sin	2813	1894	1097
god	662	398	520
hell	604	434	256
rite	549	511	265
angel	429	173	82
nun	426	114	95
christ	302	96	100
amen	283	137	103
soul	263	103	41
zen	225	131	49
informal
o	342639	220796	116229
r	262964	166676	85773
u	130336	81471	47682
b	90532	53360	28868
k	74912	41199	21658
ha	55710	22620	9410
er	54163	36641	20283
ht	34927	20090	10022
yo	22065	12445	7607
da	17346	10056	5103
nonflu
er	54674	36970	20524
ah	13515	3250	1820
um	5873	3361	1815
mm	2953	1907	1093
ugh	2227	1470	886
hm	1940	610	283
oh	1817	1286	740
uh	1154	530	243
zz	845	479	284
well	691	461	356
filler
rr	6191	3947	1578
idk	194	84	51
anyway	100	47	40
woah	38	7	3
ykn	24	11	8
dunno	20	13	5
whoa	14	9	1
youknow	12	2	2
blah	4	1	7