# Import Dataset
load("PubmedCommons.RData")

################################################
########### RESHAPE DATA FOR ANALYSIS ##########
################################################

# AUTHORS -----------------------------------------------------------------
library(reshape2)
df.author <- melt(mydata, 
                  id.vars="PMID", 
                  value.name="name",
                  variable.name="num.c",
                  measure.vars= c("author.01", "author.02", "author.03", "author.04", "author.05",
                                  "author.06", "author.07", "author.08", "author.09", "author.10",
                                  "author.11", "author.12", "author.13"), na.rm=TRUE)
df.author <- subset(df.author, name != "")

df.author$name  <- as.factor(df.author$name)

df.auth.freq1 <- data.frame(sort(table(df.author$name), decreasing=TRUE))
df.auth.freq <- data.frame(author=rownames(df.auth.freq1), comments=df.auth.freq1$sort.table.df.author.name...decreasing...TRUE.)
rm(df.auth.freq1)

rownames(df.auth.freq)=NULL
df.auth.freq$perc <- round((df.auth.freq$comments/nrow(df.author)*100),1)
df.auth.freq$cs <- cumsum(df.auth.freq$comments)
df.auth.freq$csp <- round((df.auth.freq$cs/nrow(df.author)*100),1)

tab.user <- sort(table(df.author$name), decreasing=TRUE)
df.tab.user <- data.frame(tab.user)
df.user <- data.frame(name=rownames(df.tab.user), freq=df.tab.user$tab.user)
df.user <- df.user[order(df.user$name),]
rm(df.tab.user, tab.user)

# GENDER ------------------------------------------------------------------


df.sex <- melt(mydata, 
               id.vars="PMID", 
               value.name="sex",
               variable.name="num.c",
               measure.vars= c("sex.01", "sex.02", "sex.03", "sex.04", "sex.05",
                               "sex.06", "sex.07", "sex.08", "sex.09", "sex.10",
                               "sex.11", "sex.12", "sex.13"), na.rm=TRUE)

df.sex$sex <- as.factor(df.sex$sex)

lst.date <- c("14_01", "14_02", "14_03", "14_04", "14_05")
for(i in 1:nrow(df.sex)){df.sex$launch[i] <- grepl(paste(lst.date, collapse="|"), 
                                                   df.sex$month[i], ignore.case=T)}



# AUTHOR & GENDER ---------------------------------------------------------


df.auth.sex1 <- cbind(df.author, df.sex)
df.auth.sex1 <- df.auth.sex1[!duplicated(df.auth.sex1$name),] 
df.auth.sex <- data.frame(name=df.auth.sex1$name, sex=df.auth.sex1$sex)
df.auth.sex <- df.auth.sex[order(df.auth.sex$name),]
df.auth.sex$freq <- df.user$freq
df.auth.sex$freq <- as.numeric(df.auth.sex$freq)
rm(df.auth.sex1)


# DATES -------------------------------------------------------------------

df.date <- melt(mydata, 
                id.vars="PMID", 
                value.name="date",
                variable.name="num.c",
                measure.vars= c("date.01", "date.02", "date.03", "date.04", "date.05",
                                "date.06", "date.07", "date.08", "date.09", "date.10",
                                "date.11", "date.12", "date.13"), na.rm=TRUE)
df.date <- subset(df.date, date != "")

df.date$date <- as.character(df.date$date)
library(stringr)
df.date$month <- as.factor(str_sub(df.date$date,1,8))


df.date$month <- factor(df.date$month, 
                        levels=c("2013 Jun", "2013 Jul", "2013 Aug", "2013 Sep",
                                 "2013 Oct", "2013 Nov", "2013 Dec", "2014 Jan",
                                 "2014 Feb", "2014 Mar", "2014 Apr", "2014 May"),
                        labels=c("13_06", "13_07", "13_08", "13_09",
                                 "13_10", "13_11", "13_12", "14_01",
                                 "14_02", "14_03", "14_04", "14_05"))


# WORDS, QUESTIONS ------------------------------------------------------------


library(qdap)
library(qdapTools)

df.txt <- melt(mydata, 
               id.vars="PMID", 
               value.name="txt",
               variable.name="num.c",
               measure.vars= c("txt.01", "txt.02", "txt.03", "txt.04", "txt.05",
                               "txt.06", "txt.07", "txt.08", "txt.09", "txt.10",
                               "txt.11", "txt.12", "txt.13"), na.rm=TRUE)

df.txt <- subset(df.txt, txt != "")
for(i in 1:nrow(df.txt)){df.txt$words[i] <- polarity(df.txt$txt[i])$group[3]}
for(i in 1:nrow(df.txt)){df.txt$question[i] <- grepl("\\?", df.txt$txt[i])}


df.txt$words <- as.numeric(df.txt$words)
df.txt$question <- factor(df.txt$question,
                          levels=c("FALSE", "TRUE"),
                          labels=c("no question","question"))
df.txt$sex <- df.sex$sex


# References --------------------------------------------------------------
# Return comments with hyperlinks
lst.hlink <- c("http", "www")
for(i in 1:nrow(df.txt)){df.txt$hlink[i] <- grepl(lst.hlink, df.txt$txt[i], ignore.case=T)}

df.ref <- melt(mydata, 
               id.vars="PMID", 
               value.name="ref",
               variable.name="num.c",
               measure.vars= c("ref.01", "ref.02", "ref.03", "ref.04", "ref.05",
                               "ref.06", "ref.07", "ref.08", "ref.09", "ref.10",
                               "ref.11", "ref.12", "ref.13"), na.rm=TRUE)

df.txt$lit <- df.ref$ref

df.txt$ref <- ifelse(df.txt$hlink==TRUE | df.txt$lit==1,1,0)

df.txt$ref <- factor(df.txt$ref, levels=c(0,1), labels=c("no references", "references"))



# Return Journals with specific key words in title ------------------------

# Journals: Psychiatry & Psychology
for(i in 1:nrow(mydata)){mydata$psych[i] <- grepl("psych", mydata$journal[i], ignore.case=T)}

# Journals: Pediatr
for(i in 1:nrow(mydata)){mydata$pedia[i] <- grepl("pediat", mydata$journal[i], ignore.case=T)}

# Journals: Brain
for(i in 1:nrow(mydata)){mydata$brain[i] <- grepl("brain", mydata$journal[i], ignore.case=T)}

# Journals: Neurosci & Neurol
for(i in 1:nrow(mydata)){mydata$neuro[i] <- grepl("neuro", mydata$journal[i], ignore.case=T)}

# Journals: Patho
for(i in 1:nrow(mydata)){mydata$patho[i] <- grepl("patho", mydata$journal[i], ignore.case=T)}

# Journals: Oncol & Cancer
lst.onco <- c("oncol", "cancer")
for(i in 1:nrow(mydata)){mydata$onco[i] <- grepl(paste(lst.onco, collapse="|"), 
                                                 mydata$journal[i], ignore.case=T)}

# Journals: heart, cardiovasc, cardiol
lst.cardio <- c("cardio", "heart")
for(i in 1:nrow(mydata)){mydata$cardio[i] <- grepl(paste(lst.cardio, collapse="|"), 
                                                   mydata$journal[i], ignore.case=T)}

# Journals: intern, surg
lst.intern <- c("intern", "surg")
for(i in 1:nrow(mydata)){mydata$intern[i] <- grepl(paste(lst.intern, collapse="|"), 
                                                   mydata$journal[i], ignore.case=T)}

# Journals: biol, mol, cell, genet, chem
lst.biol <- c("biol", "mol", "cell")
for(i in 1:nrow(mydata)){mydata$biol[i] <- grepl(paste(lst.biol, collapse="|"), 
                                                 mydata$journal[i], ignore.case=T)}

# Journals: Biol
for(i in 1:nrow(mydata)){mydata$bio[i] <- grepl("biol", mydata$journal[i], ignore.case=T)}

# Journals: Chem
for(i in 1:nrow(mydata)){mydata$chem[i] <- grepl("chem", mydata$journal[i], ignore.case=T)}

# Journals: Genetic
for(i in 1:nrow(mydata)){mydata$genet[i] <- grepl("genet", mydata$journal[i], ignore.case=T)}



#################################
############ ANALYSIS ###########
#################################


# Freq Stats --------------------------------------------------------------

table(mydata$biol)
table(mydata$gen)
table(mydata$neuro)
table(mydata$intern)
table(mydata$psych)
table(mydata$cardio)
table(mydata$onco)
table(mydata$chem)
table(mydata$genet)
table(mydata$pedia)
table(mydata$patho)
table(mydata$brain)

summary(df.txt$words)
sd(df.txt$words)
table(df.txt$question)



# t-Tests -----------------------------------------------------------------


# Compare between comments with questions vs. no questions
t.test(words ~ question, data = df.txt)


# Compare between female and male contributers
t.test(words ~ sex, data = subset(df.txt, sex != "dk"))


# Chi-Squared -------------------------------------------------------------
df.qs <- subset(df.txt, sex!="dk")
df.qs$sex <- factor(df.qs$sex)
chisq.test(table(df.qs$question, df.qs$sex))

library(Hmisc)

# AUTORS -----------------------------------------------------------------

describe(mydata$num.c)

# GENDER ------------------------------------------------------------------

describe(df.sex$sex)

# AUTHOR & GENDER ---------------------------------------------------------

table(df.auth.sex$sex)
describe(df.auth.sex$sex)

# Compare between female and male contributers
t.test(freq ~ sex, data = subset(df.auth.sex, sex != "dk"))
tapply(df.auth.sex$freq, df.auth.sex$sex, median)

# Year of Publication -----------------------------------------------------

table(mydata$year)

# DATES -------------------------------------------------------------------

lp <- plot(table(df.date$month), 
           type="b",
           las=1,
           pch=c(4, 4, 4, 4, 2, 2, 2, 19, 19, 19, 19, 19),
           lty=c(1, 1, 1, 0, 1, 1, 0, 1, 1, 1,0),
           cex.lab=0.75,
           cex.axis=0.6,
           ylim=c(0,250),
           ylab="number of comments"
)
grid()
box()
text(lp, 
     (table(df.date$month)+5), 
     table(df.date$month), 
     cex=0.75,
     font=2,
     pos=3) # Position der Zahlen
text(2.5, 75, "testing version", cex=0.75)
text(6.5, 175, "pilot version", cex=0.75)
text(10, 225, "public version", cex=0.75)


# Questions, References, Hyperlinks ---------------------------------------

table(df.txt$question)
table(df.txt$ref)
table(df.txt$hlink)

# CLEANSING ---------------------------------------------------------------
remove(i, lp, lst.hlink, lst.date, lst.biol, lst.cardio, lst.intern, lst.onco)