library(shinydashboard)
library(RMySQL)
#library(sqldf)
library(feather)
library(ggplot2)
library(dplyr)
library(reshape2)
library(lubridate)
library(plotly) 

# 1. Data preprocessing ------------------------------------------------------

path_temp_files <- "data"

# 1.1. Load from web db ---------------------------------------------------

load_web_db <- function(){
  
  library(RMySQL)
  
  # 1.1.1. Connect db
  db <-  dbConnect(MySQL(), user='analyst', password='exa_analyst1&',
                   dbname='esi_management',
                   host='lecanaldb.c12hbxfn3xzn.eu-west-1.rds.amazonaws.com',
                   port=3306)
  
  # 1.1.2. load one table
  db_table_load <- function(table){
    # read
    rs = dbSendQuery(db, paste0("select * from ", table))
    df = fetch(rs, n=-1)
    write_feather(df, paste0(path_temp_files, "/", table))
    
  }
  
  # Delete directory for temp files if exists
  if(dir.exists(path_temp_files)){
    unlink(path_temp_files, recursive = TRUE, force = TRUE)  
  }
  
  # Create new directory
  dir.create(path_temp_files)
  
  # 1.1.3. Load tables
  df_entity <- db_table_load("wp_esi_entity")
  df_tag <- db_table_load("wp_esi_tag")
  df_tag_entity <- db_table_load("wp_esi_tag_entity")
  df_investment <- db_table_load("wp_esi_investment")
  
  # 1.1.4. disconnect
  dbDisconnect(db)
  
  # 1.1.5 packages management
  detach("package:RMySQL", unload=TRUE)
  library(sqldf)
  
}

# Check directory for cached files
exist_cash <- function(){
  return(dir.exists(path_temp_files))
}

read_df <- function(name){
  return(read_feather(paste0(path_temp_files, "/", name)))
}

# Read data from cash
if (exist_cash()){
  df_entity <- read_df("wp_esi_entity")
  df_tag <- read_df("wp_esi_tag")
  df_tag_entity <- read_df("wp_esi_tag_entity")
  df_investment <- read_df("wp_esi_investment")
  
  # packages management
  detach("package:RMySQL", unload=TRUE)
  library(sqldf)
  
} else {
  
  load_web_db()
  
  # todo: remove copy/paste  
  df_entity <- read_df("wp_esi_entity")
  df_tag <- read_df("wp_esi_tag")
  df_tag_entity <- read_df("wp_esi_tag_entity")
  df_investment <- read_df("wp_esi_investment")
  
}


# UI ----------------------------------------------------------------------

sidebar <- dashboardSidebar(
  sidebarMenu(
    menuItem("About", tabName = "about", icon = icon("dashboard")),
    menuItem("Companies", icon = icon("dashboard"),
             menuSubItem("Records all", tabName = "q0_1"),
             menuSubItem("Records by tags", tabName = "q0_2"),
             p(),
             menuSubItem("By countries", tabName = "q1"),
             menuSubItem("By tags", tabName = "q3"),
             menuSubItem("By size per tags", tabName = "q4"),
             menuSubItem("By age per tags", tabName = "q5")             
             ),
    menuItem("Investments", icon = icon("money"),
             menuSubItem("By tags", tabName = "q2"),
             menuSubItem("By tags, date and funding", tabName = "q6")
             )
  )
)

body <- dashboardBody(
  tabItems(
    tabItem(tabName = "about",
            htmlOutput("system_info"),
            actionButton("btn_load_db", "Update data"),
            htmlOutput("txt_process")
    ),
    tabItem(tabName = "q0_1",
            fluidPage(
              fluidRow(DT::dataTableOutput("q0_1_table"))
            )),
    tabItem(tabName = "q0_2",
            fluidPage(
              fluidRow(selectizeInput('q0_2_tags', 'Tags', choices = df_tag$name, multiple = TRUE)),
              fluidRow(radioButtons("q0_2_tags_filter", "Tags filter",
                       choices = list("OR" = 1, "AND" = 2), selected = 1, inline = T)),
              fluidRow(DT::dataTableOutput("q0_2_table")),
              fluidRow(downloadButton('btn_download_q0_2', 'Download'))
            )),
    tabItem(tabName = "q1",
            fluidPage(
              fluidRow(h2("Distribution of AI companies by countries")),
              fluidRow(p(tags$b("Number of companies"))),
              fluidRow(
                column(3, textInput("q1_entities_min", "Min", "100")),
                column(3, textInput("q1_entities_max", "Max", "10000"))),
              fluidRow(plotlyOutput("q1")),
              fluidRow(DT::dataTableOutput("q1_table")),
              fluidRow(downloadButton('btn_download_q1', 'Download'))
            )
            
    ),
    tabItem(tabName = "q2",
            fluidPage(
              fluidRow(h2("Investments per tags")),
              fluidRow(selectInput("q2_choice", "Total raised",
                        c("<     1,000,000",
                          "      1,000,000 :    10,000,000",
                          "     10,000,000 :   100,000,000",
                          "    100,000,000 : 1,000,000,000",
                          "> 1,000,000,000"))
              ),
              fluidRow(plotlyOutput("q2")),
              fluidRow(DT::dataTableOutput("q2_table")),
              fluidRow(downloadButton('btn_download_q2', 'Download'))
            )
            
    ),
    tabItem(tabName = "q3",
            fluidPage(
              fluidRow(h2("Number of companies per tags")),
              fluidRow(selectInput("q3_choice", "Number of tags",
                                   c("> 100", "50:100", "10:50", "5:10", "3:5", "1:3", "1: Max"))),
              fluidRow(plotlyOutput("q3")),
              fluidRow(DT::dataTableOutput("q3_table")),
              fluidRow(downloadButton('btn_download_q3', 'Download'))
            )
            
    ),
    tabItem(tabName = "q4",
            fluidPage(
              fluidRow(h2("Average size of company per tags")),
              fluidRow(selectInput("q4_choice", "Average size",
                                   c("> 1000", "500 : 1000", "250 : 500", "100 : 250", "50 : 100", "25 : 50", "1 : 25", "1 : Max"))),
              fluidRow(plotlyOutput("q4")),
              fluidRow(DT::dataTableOutput("q4_table")),
              fluidRow(downloadButton('btn_download_q4', 'Download'))
            )),
    tabItem(tabName = "q5",
            fluidPage(
              fluidRow(h2("Average age of company per tags")),
              fluidRow(selectInput("q5_choice", "Average age",
                                   c("> 20", "15 : 20", "10 : 15", "7 : 10", "5 : 7", "3 : 5", "1 : 3", "1 : Max"))),
              fluidRow(plotlyOutput("q5")),
              fluidRow(DT::dataTableOutput("q5_table")),
              fluidRow(downloadButton('btn_download_q5', 'Download'))
            )),
    tabItem(tabName = "q6",
            fluidPage(
              fluidRow(selectizeInput('q6_tags', 'Tags', choices = df_tag$name, multiple = TRUE)),
              fluidRow(radioButtons("q6_tags_filter", "Tags filter",
                                    choices = list("OR" = 1, "AND" = 2), selected = 1, inline = T)),
              fluidRow(
                dateRangeInput("q6_dates", label = h3("Date range"),
                                            start = min(as.Date(df_investment$investment_date)), 
                                            end = max(as.Date(df_investment$investment_date)))),
              fluidRow(selectInput("q6_amount", "Amount invested",
                                   c("<     1,000,000",
                                     "      1,000,000 :    10,000,000",
                                     "     10,000,000 :   100,000,000",
                                     "    100,000,000 : 1,000,000,000",
                                     "> 1,000,000,000"))),
              fluidRow(DT::dataTableOutput("q6_table")),
              fluidRow(downloadButton('btn_download_q6', 'Download'))
            ))
    
  )
)


ui <- dashboardPage(
  dashboardHeader(title = "ESI Analytics"),
  sidebar,
  body
)

# SERVER ------------------------------------------------------------------

server <- function(input, output, session) {

# System info -------------------------------------------------------------  

output$system_info <- renderUI({
  HTML(paste0(
    "<p><b>General information</b>: </b></p>",
    "<p>Entities: <b>", nrow(df_entity), "</b></p>",
    "<p>Tags: <b>", nrow(df_tag), "</b></p>"
  ))
  
})

# Load
observeEvent(input$btn_load_db, {
  # 1. do it
  load_web_db()
  # todo: remove copy/paste
  df_entity <- read_df("wp_esi_entity")
  df_tag <- read_df("wp_esi_tag")
  df_tag_entity <- read_df("wp_esi_tag_entity")
  df_investment <- read_df("wp_esi_investment")
  
  # 3. show message
  output$txt_process <- renderUI({
    HTML("Database updated")
  }) 
  
})

# q0_1: All records from entity ---------------------------------------------

# 1. Data frame
df_q0_1 <- reactive({
  df_entity %>% 
    mutate(
      country = ifelse(country=="", "N/A", country),
      description = paste0(substr(description, 1, 10), "...")) %>% 
    subset(select = c(name, url, description, country, city, founded_in, employees, total_raised))
})

# 2. Table
output$q0_1_table <- DT::renderDataTable(
  DT::datatable(df_q0_1(), options = list(pageLength = 25), rownames= FALSE)
)

# q0_2: Records from entity by tags---------------------------------------------

# 1. Table
output$q0_2_table <- DT::renderDataTable(
  DT::datatable(df_q0_2(), options = list(pageLength = 10, searching = TRUE), rownames= FALSE)
)

# 2. Data frame
df_q0_2 <- reactive({
  
  # 1. get list of selected tags from input
  list_of_selected_tags <- input$q0_2_tags
  
  # 2. pipeline
  tags_selected <-  df_tag %>% 
    filter(name %in% list_of_selected_tags)
  
  # 3. get entities with tags
  entities_tags_selected <- df_tag_entity %>% 
    filter(tag_id %in% tags_selected$id)
  
  #write_feather(entities_tags_selected, "entities_tags_selected")
  
  # list of unique tags
  unique_tags <- unique(entities_tags_selected$tag_id)
  
  #
  # Check type of filter
  #
  # 4.1. Select entities <OR>
  if (input$q0_2_tags_filter == "1"){
    # Selection
    entities_selected <- df_entity %>% 
      filter(id %in% entities_tags_selected$entity_id)
    
    # Feature Engineering
    entities_selected <- entities_selected %>% 
      mutate(
        country = ifelse(country=="", "N/A", country),
        description = paste0(substr(description, 1, 20), "...")) %>% 
      subset(select = c(name, url, description, country, city, founded_in, employees, total_raised))
  }
  
  # 4.2. Select entities <AND>
  if (input$q0_2_tags_filter == "2"){
    
    # Select
    entities_selected <- entities_tags_selected %>% 
      group_by(entity_id) %>% 
      summarise(n = n()) %>% 
      filter(n == length(unique_tags))
    
    # Selected entities
    entities_selected_ids <- unique(entities_selected$entity_id)

    # Feature Engineering
    entities_selected <- df_entity %>%
      filter(id %in% entities_selected_ids) %>%
      mutate(
        country = ifelse(country=="", "N/A", country),
        description = paste0(substr(description, 1, 20), "...")) %>% 
      subset(select = c(name, url, description, country, city, founded_in, employees, total_raised))
  }
  
  # 5. Get result
  return(entities_selected)
  
})

# Download
output$btn_download_q0_2 <- downloadHandler(
  filename = "companies_by_tags.csv",
  content = function(file) {
    write.csv(df_q0_2(), file)
  }
)

# Q1: Distribution of AI companies by countries --------------------------

  # 1. Range of companies from min to max 
  q1_get_min <- renderText({ input$q1_entities_min })
  q1_get_max <- renderText({ input$q1_entities_max })
  
  # 1. Data frame
  df_q1 <- reactive({
    df_entity %>% 
      group_by(country) %>% 
      summarise(sum = n()) %>% 
      arrange(-sum) %>% 
      mutate(country = ifelse(country=="", "N/A", country))  
  })
  
  # 2. Table
  output$q1_table <- DT::renderDataTable(
    DT::datatable(df_q1(),  options = list(pageLength = 10)) # , searching = FALSE
  )
  
  # 3. Graph
  output$q1 <- renderPlotly({
  
    gg_title <- paste0("Countries with ", q1_get_min(), " : ", q1_get_max(),  " companies")
    
    gg_q1 <- df_q1() %>% 
      filter(sum %in% q1_get_min():q1_get_max()) %>% 
      ggplot(aes(reorder(country, sum), sum, fill=factor(sum))) +
      ggtitle(gg_title) +
      xlab("Countries") +
      ylab("Number of companies") +  
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()
    
    # Return result
    gg_q1
    
  })
  
  # Download
  output$btn_download_q1 <- downloadHandler(
    filename = "q1_entities_by_countries.csv",
    content = function(file) {
      write.csv(df_q1(), file)
    }
  )
    
# Q2: Investments per tags --------------------------
  
  # 1. Data frame
  df_q2 <- reactive({
    
    # data file for Q2 question
    q2_data_file <- paste0(path_temp_files, "/q2_data")
    
    if(file.exists(q2_data_file)){
      df_res_final <- read_feather(q2_data_file)
    }  else {
      
      # Q2.1. Join data
      s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
      s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
      x <- sqldf(s1)
      
      x <- sqldf(s2)
      
      # Q2.2. Change column names
      colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                       "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                       "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                       "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                       "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
      
      # Q2.3. Select features
      x <- x %>% 
        subset(select = c("entity_id", "total_raised", "tag_id")) %>% 
        filter(!is.na(tag_id))
      
      # Q2.4. Transform data set
      xt <- dcast(x, tag_id ~ total_raised)
      
      # Q2.5. Create tags column
      xt_tags <- xt %>% 
        subset(select = c("tag_id"))
      
      # Q2.6. For one tag function
      one_tag <- function(id){
        # 1. basic data set
        xt_case <- xt %>% 
          filter(tag_id == id) %>%
          subset(select=2:length(colnames(xt))) %>% 
          t()
        
        # 2. add "total_raised"
        rnames <- as.data.frame(row.names(xt_case))
        colnames(rnames) <- c("total_raised")
        
        xt_case <- cbind(rnames, xt_case)
        row.names(xt_case) <- NULL
        
        colnames(xt_case) <- c("total_raised", "sum_entities")
        
        xt_case <- xt_case %>% 
          mutate(total_raised = as.numeric(as.character(total_raised))) %>% 
          filter(!is.na(total_raised)) %>% 
          mutate(sum_total_raised = total_raised * sum_entities) %>% 
          subset(select = c("sum_total_raised"))
        
        return(xt_case)
      } # one_tag
      
      # Q2.7. init result data set
      z <- one_tag(unique(xt_tags$tag_id)[1])  
      colnames(z) <- c(unique(xt_tags$tag_id)[1])
      
      # 1. Progress object
      progress_q2 <- shiny::Progress$new()
      on.exit(progress_q2$close())
      progress_q2$set(message = "Processing", value = 0)
      
      # Q2.8. for loop
      end_for <- unique(xt_tags$tag_id)[2:length(unique(xt_tags$tag_id))]
      for(i in end_for){
        # Show progress
        progress_q2$inc(1/length(end_for), detail = paste("", i))
        
        tmp <- one_tag(i)
        colnames(tmp) <- c(i)
        z <- cbind(z, tmp)
      }
      
      # Q2.9. transpose
      z <- t(z)
      
      # Q2.10. sum total_raised
      xt_sum_total_raised <- as.data.frame(z) %>% 
        mutate(sum_total_raised = rowSums(.)) %>% 
        subset(select = c("sum_total_raised"))
      
      # Q2.11. Merge data sets
      xt_res <- cbind(xt_tags, xt_sum_total_raised)
      
      # Q2.12. Add tag names
      s <- "select xt_res.*, df_tag.* from xt_res left join df_tag on xt_res.tag_id = df_tag.id"
      df_res_final <- sqldf(s)
      
      df_res_final <- df_res_final %>% 
        subset(select = c("tag_id", "name", "sum_total_raised"))
      
      # Cash files
      write_feather(df_res_final, q2_data_file)

    }    
  
    # get result
    return(df_res_final)
    
  })
  
  # 2. Table
  output$q2_table <- DT::renderDataTable(
    DT::datatable(df_q2(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_q2 <- downloadHandler(
    filename = "q2_total_raised_by_tags.csv",
    content = function(file) {
      write.csv(df_q2(), file)
    }
  )
  
  # Draw plot
  output$q2 <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$q2_table_rows_selected
    if(length(selected_rows) > 0){
      df_q2_plot <- df_q2()[selected_rows, ]
    } else {
      df_q2_plot <- df_q2()  
    }

    #
    # Q2.13. Draw graphs
    #
    
    # > 1,000,000,000
    gg_q2_5 <- df_q2_plot %>%
      filter(sum_total_raised > 1000000000) %>%
      ggplot(aes(reorder(name, sum_total_raised), sum_total_raised, fill=factor(sum_total_raised))) +
      ggtitle("Total raised of companies per Tag (> 1,000,000,000)") +
      xlab("") +
      ylab("Total raised of company") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()

    # 100,000,000 : 1,000,000,000
    gg_q2_4 <- df_q2_plot %>%
      filter((sum_total_raised > 100000000) &  (sum_total_raised <= 1000000000)) %>%
      ggplot(aes(reorder(name, sum_total_raised), sum_total_raised, fill=factor(sum_total_raised))) +
      ggtitle("Total raised of companies per Tag (100,000,000 : 1,000,000,000)") +
      xlab("") +
      ylab("Total raised of company") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()

    # 10,000,000 : 100,000,000
    gg_q2_3 <- df_q2_plot %>%
      filter((sum_total_raised > 10000000) &  (sum_total_raised <= 100000000)) %>%
      ggplot(aes(reorder(name, sum_total_raised), sum_total_raised, fill=factor(sum_total_raised))) +
      ggtitle("Total raised of companies per Tag (10,000,000 : 100,000,000)") +
      xlab("") +
      ylab("Total raised of company") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()

    # 1,000,000 : 10,000,000
    gg_q2_2 <- df_q2_plot %>%
      filter((sum_total_raised > 1000000) &  (sum_total_raised <= 10000000)) %>%
      ggplot(aes(reorder(name, sum_total_raised), sum_total_raised, fill=factor(sum_total_raised))) +
      ggtitle("Total raised of companies per Tag (1,000,000 : 10,000,000)") +
      xlab("") +
      ylab("Total raised of company") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()
    
    # < 1,000,000
    gg_q2_1 <- df_q2_plot %>%
      filter((sum_total_raised <= 1000000)) %>%
      ggplot(aes(reorder(name, sum_total_raised), sum_total_raised, fill=factor(sum_total_raised))) +
      ggtitle("Total raised of companies per Tag (< 1,000,000)") +
      xlab("") +
      ylab("Total raised of company") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()
    
    # Select graph
    if(input$q2_choice == "<     1,000,000")                { g <- gg_q2_1 }
    if(input$q2_choice == "      1,000,000 :    10,000,000"){ g <- gg_q2_2 }
    if(input$q2_choice == "     10,000,000 :   100,000,000"){ g <- gg_q2_3 }
    if(input$q2_choice == "    100,000,000 : 1,000,000,000"){ g <- gg_q2_4 }
    if(input$q2_choice == "> 1,000,000,000")                { g <- gg_q2_5 }
    
    # Return result
    g
    
  })
    
# Q3: Number of companies per tags --------------------------
  
  # 1. Data frame
  df_q3 <- reactive({
    # Q3.1. Join data
    s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
    s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
    x <- sqldf(s1)
    x <- sqldf(s2)
    
    colnames(x)
    # Q3.2. Change column names
    colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                     "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                     "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                     "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                     "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
    
    # Q3.3. Select features
    x <- x %>% 
      subset(select = c("entity_id", "total_raised", "tag_id", "tag_name"))
    
    # Q3.4. Transform data set
    xt <- dcast(x, entity_id ~ tag_id)
    
    xt[!is.na(xt)] <- 1
    
    # change column types
    xt[ , colnames(xt)] <-  apply(xt[ , colnames(xt)], 2, function(x) as.numeric(as.character(x)))
    
    # calculate sums
    xt["total", ] <- colSums(xt, na.rm = T)
    xt <- xt["total", ]
    
    # remove feature
    xt$entity_id <- NULL
    
    # transpose
    xt <- t(xt)
    
    # add "tag_id"
    rnames <- as.data.frame(row.names(xt))
    colnames(rnames) <- c("tag_id")
    
    # merge
    xt <- cbind(rnames, xt)
    
    # change row names
    row.names(xt) <- NULL
    
    # add tag names
    xt <- sqldf("select xt.*, df_tag.* from xt left join df_tag on xt.tag_id = df_tag.id")
    
    # select features and arrange
    xt <- xt %>% 
      subset(select = c("name", "total")) %>% 
      arrange(-total)
    
    # get result
    return(xt)
    
  })
  
  # 2. Table
  output$q3_table <- DT::renderDataTable(
    DT::datatable(df_q3(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_q3 <- downloadHandler(
    filename = "q3_companies_by_tags.csv",
    content = function(file) {
      write.csv(df_q3(), file)
    }
  )
  
  # Draw plot
  output$q3 <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$q3_table_rows_selected
    if(length(selected_rows) > 0){
      df_q3_plot <- df_q3()[selected_rows, ]
    } else {
      df_q3_plot <- df_q3()  
    }
    
    #
    # Q2.13. Draw graphs
    #
    
    # Select graph
    
    # 1:max companies    
    if(input$q3_choice == "1: Max"){
      gg_q3 <- df_q3_plot %>% 
        filter(total %in% 1:max(df_q3_plot$total)) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (1 : Max)")
    }
    
    # > 100 companies    
    if(input$q3_choice == "> 100"){
      gg_q3 <- df_q3_plot %>% 
        filter(total > 100) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (> 100)")
    }
    
    # 50:100
    if(input$q3_choice == "50:100"){
      gg_q3 <- df_q3_plot %>% 
        filter(total %in% 50:100) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (50-100)")
    }
    
    # 10:50
    if(input$q3_choice == "10:50"){
      gg_q3 <- df_q3_plot %>% 
        filter(total %in% 10:50) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (10-50)")
    }
    
    
    # 1:10
    if(input$q3_choice == "5:10"){
      gg_q3 <- df_q3_plot %>% 
        filter(total %in% 5:10) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (5-10)")
    }
    
    # 3:5
    if(input$q3_choice == "3:5"){
      gg_q3 <- df_q3_plot %>% 
        filter(total %in% 3:5) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (3-5)")
    }
    
    # 1:3
    if(input$q3_choice == "1:3"){
      gg_q3 <- df_q3_plot %>% 
        filter(total %in% 1:3) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (1-3)")
    }
    
    
    # the same for all graphs
    gg_q3 <- gg_q3 +
      xlab("") +
      ylab("Number of companies") +  
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      theme_bw() +
      coord_flip()

    # Return result
    gg_q3
    
  })
  
# Q4: Average size of companies per tags --------------------------  
  df_q4 <- reactive({
    # Q4.1. Join data
    s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
    s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
    x <- sqldf(s1)
    x <- sqldf(s2)
    
    # Q4.2. Change column names
    colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                     "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                     "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                     "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                     "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
    
    # Q4.3. Select features
    x <- x %>% 
      subset(select = c("entity_id", "employees", "tag_id")) %>% 
      filter(!is.na(tag_id))
    
    # Q3.4. Transform data set
    xt <- dcast(x, tag_id ~ employees)
    
    # Q3.6. Create tags column
    xt_tags <- xt %>% 
      subset(select = c("tag_id"))
    
    #
    # Q3.7. Create "sum entities"
    #
    xt_sum_entities <- xt %>% 
      subset(select=2:length(colnames(xt))) %>% 
      mutate(sum_entities = rowSums(.)) %>% 
      subset(select = c("sum_entities"))
    
    #
    # Q3.8. Create "sum employees"
    #
    
    # For one tag function
    one_tag <- function(id){
      # 1. basic data set
      xt_case <- xt %>% 
        filter(tag_id == id) %>%
        subset(select=2:length(colnames(xt))) %>% 
        t()
      
      # 2. add "employees"
      rnames <- as.data.frame(row.names(xt_case))
      colnames(rnames) <- c("employees")
      
      xt_case <- cbind(rnames, xt_case)
      row.names(xt_case) <- NULL
      
      colnames(xt_case) <- c("employees", "sum_entities")
      
      xt_case <- xt_case %>% 
        mutate(employees = as.numeric(as.character(xt_case$employees)),
               sum_employees = employees * sum_entities) %>% 
        subset(select = c("sum_employees"))
      
      return(xt_case)
    }
    
    # init result data set
    z <- one_tag(unique(xt_tags$tag_id)[1])  
    colnames(z) <- c(unique(xt_tags$tag_id)[1])
    
    # for loop
    end_for <- unique(xt_tags$tag_id)[2:length(unique(xt_tags$tag_id))]
    for(i in end_for){
      tmp <- one_tag(i)
      colnames(tmp) <- c(i)
      z <- cbind(z, tmp)
    }
    
    # transpose
    z <- t(z)
    
    # 1. sum employees
    xt_sum_employees <- as.data.frame(z) %>% 
      mutate(sum_employees = rowSums(.)) %>% 
      subset(select = c("sum_employees"))
    
    
    # Q3.9. Merge data sets
    xt_res <- cbind(xt_tags, xt_sum_entities)
    xt_res <- cbind(xt_res, xt_sum_employees)
    
    # Q3.10. Calculate average size
    xt_res <- xt_res %>%  
      mutate(sum_entities = as.numeric(sum_entities),
             sum_employees = as.numeric(sum_employees),
             avr_size = round(sum_employees / sum_entities)) %>% 
      subset(select = c("tag_id", "avr_size"))
    
    # Q3.10. Add tag names
    
    s <- "select xt_res.*, df_tag.* from xt_res left join df_tag on xt_res.tag_id = df_tag.id"
    df_res_final <- sqldf(s)
    
    df_res_final <- df_res_final %>% 
      subset(select = c("tag_id", "name", "avr_size"))
    
    # get result
    return(df_res_final)
    
  })
  
  # 2. Table
  output$q4_table <- DT::renderDataTable(
    DT::datatable(df_q4(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_q4 <- downloadHandler(
    filename = "q4_average_size_companies_by_tags.csv",
    content = function(file) {
      write.csv(df_q4(), file)
    }
  )
  
  # Draw plot
  output$q4 <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$q4_table_rows_selected
    if(length(selected_rows) > 0){
      df_q4_plot <- df_q4()[selected_rows, ]
    } else {
      df_q4_plot <- df_q4()  
    }
    
    # 1:max companies    
    if(input$q4_choice == "1 : Max"){
      gg_q4 <- df_q4_plot %>%
        filter(avr_size %in% 1:max(df_q4_plot$avr_size)) %>%
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per tags (1 : Max)")
    }
    
    # > 1000
    if(input$q4_choice == "> 1000"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size > 1000) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (> 1000)")
    }
    
    # 500 : 1000
    if(input$q4_choice == "500 : 1000"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size %in% 500:1000) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (500 : 1000)")
    }
    
    # 250 : 500
    if(input$q4_choice == "250 : 500"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size %in% 250:500) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (250 : 500)")
    }
    
    # 100 : 250
    if(input$q4_choice == "100 : 250"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size %in% 100:250) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (100 : 250)")
    }
    
    # 50 : 100
    if(input$q4_choice == "50 : 100"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size %in% 50:100) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (50 : 100)")
    }    
    
    # 25 : 50
    if(input$q4_choice == "25 : 50"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size %in% 25:50) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (25 : 50)")
    }    
    
    # 1 : 25
    if(input$q4_choice == "1 : 25"){
      gg_q4 <- df_q4_plot %>% 
        filter(avr_size %in% 1:25) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (1 : 25)")
    }    
    
    # the same for all graphs
    gg_q4 <- gg_q4 +
      xlab("") +
      ylab("Average number of employees") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      theme_bw() +
      coord_flip()
    
    # Return result
    gg_q4
    
  })

# Q5: Average age of companies per tags --------------------------  
  
df_q5 <- reactive({
  # Q5.1. Join data
  s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
  s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
  x <- sqldf(s1)
  x <- sqldf(s2)
  
  # Q5.2. Change column names
  colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                   "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                   "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                   "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                   "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
  
  # Q5.3. Check if a value is the date
  is_date <- function(mydate, date.format = "%Y-%m-%d") {
    tryCatch(!is.na(as.Date(mydate, date.format)),  
             error = function(err) {FALSE})  
  }
  
  # fix data bugs
  # x <- x %>% 
  #   filter(! is_date(founded_in)) %>% 
  #   subset(select = c("entity_id", "name", "founded_in")) %>% 
  #   unique() %>% 
  #   write.csv("bugs_founded_in.csv", row.names=F)
  
  # Q5.4. Select features
  # year > 1900
  x <- x %>% 
    subset(select = c("entity_id", "founded_in", "tag_id")) %>% 
    filter(!is.na(tag_id) & is_date(founded_in) & (year(as.Date(founded_in))>1900)) %>% 
    mutate(age = interval(as.Date(founded_in), today()) / years(1)) %>% 
    mutate(age = round(age, 2)) %>% 
    subset(select = c("entity_id", "age", "tag_id"))
  
  # Q5.5. Transform data set
  xt <- dcast(x, tag_id ~ age)
  
  # Q5.6. Create tags column
  xt_tags <- xt %>% 
    subset(select = c("tag_id"))
  
  #
  # Q5.7. Create "sum entities"
  #
  xt_sum_entities <- xt %>% 
    subset(select=2:length(colnames(xt))) %>% 
    mutate(sum_entities = rowSums(.)) %>% 
    subset(select = c("sum_entities"))
  
  #
  # Q5.8. Create "sum age" # instead 'employees'
  #
  
  # For one tag function
  one_tag <- function(id){
    # 1. basic data set
    xt_case <- xt %>% 
      filter(tag_id == id) %>%
      subset(select=2:length(colnames(xt))) %>% 
      t()
    
    # 2. add "age"
    rnames <- as.data.frame(row.names(xt_case))
    colnames(rnames) <- c("age")
    
    xt_case <- cbind(rnames, xt_case)
    row.names(xt_case) <- NULL
    
    colnames(xt_case) <- c("age", "sum_entities")
    
    xt_case <- xt_case %>% 
      mutate(age = as.numeric(as.character(xt_case$age)),
             sum_age = age * sum_entities) %>% 
      subset(select = c("sum_age"))
    
    return(xt_case)
  }
  
  # init result data set
  z <- one_tag(unique(xt_tags$tag_id)[1])  
  colnames(z) <- c(unique(xt_tags$tag_id)[1])
  
  # for loop
  end_for <- unique(xt_tags$tag_id)[2:length(unique(xt_tags$tag_id))]
  for(i in end_for){
    tmp <- one_tag(i)
    colnames(tmp) <- c(i)
    z <- cbind(z, tmp)
  }
  
  # transpose
  z <- t(z)
  
  # 1. sum age
  xt_sum_age <- as.data.frame(z) %>% 
    mutate(sum_age = rowSums(.)) %>% 
    subset(select = c("sum_age"))
  
  
  # Q3.9. Merge data sets
  xt_res <- cbind(xt_tags, xt_sum_entities)
  xt_res <- cbind(xt_res, xt_sum_age)
  
  # Q3.10. Calculate average age
  xt_res <- xt_res %>%  
    mutate(sum_entities = as.numeric(sum_entities),
           sum_age = as.numeric(sum_age),
           avr_age = round(sum_age / sum_entities), 2) %>% 
    subset(select = c("tag_id", "avr_age"))
  
  # Q3.10. Add tag names
  
  s <- "select xt_res.*, df_tag.* from xt_res left join df_tag on xt_res.tag_id = df_tag.id"
  df_res_final <- sqldf(s)
  
  df_res_final <- df_res_final %>% 
    subset(select = c("tag_id", "name", "avr_age"))

  # get result
  return(df_res_final)
    
})
  
  # 2. Table
  output$q5_table <- DT::renderDataTable(
    DT::datatable(df_q5(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_q5 <- downloadHandler(
    filename = "q5_average_age_companies_by_tags.csv",
    content = function(file) {
      write.csv(df_q5(), file)
    }
  )
  
  # Draw plot
  output$q5 <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$q5_table_rows_selected
    if(length(selected_rows) > 0){
      df_q5_plot <- df_q5()[selected_rows, ]
    } else {
      df_q5_plot <- df_q5()  
    }
    
    # 1:max companies    
    if(input$q5_choice == "1 : Max"){
      gg_q5 <- df_q5_plot %>%
        filter(avr_age %in% 1:max(df_q5_plot$avr_age)) %>%
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (1 : Max)")
    }
    
    # > 20
    if(input$q5_choice == "> 20"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age > 20) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (> 20)")
    }
    
    # 15 : 20
    if(input$q5_choice == "15 : 20"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age %in% 15:20) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (15 : 20)")
    }
    
    # 10 : 15
    if(input$q5_choice == "10 : 15"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age %in% 10:15) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (10 : 15)")
    }
    
    # 7 : 10
    if(input$q5_choice == "7 : 10"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age %in% 7:10) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (7 : 10)")
    }

    # 5 : 7
    if(input$q5_choice == "5 : 7"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age %in% 5:7) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (5 : 7)")
    }
    
    # 3 : 5
    if(input$q5_choice == "3 : 5"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age %in% 3:5) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (3 : 5)")
    }
    
    # 1 : 3
    if(input$q5_choice == "1 : 3"){
      gg_q5 <- df_q5_plot %>% 
        filter(avr_age %in% 1:3) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (1 : 3)")
    }
    
    # the same for all graphs
    gg_q5 <- gg_q5 +
      xlab("") +
      ylab("Average age of companies") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      theme_bw() +
      coord_flip()
    
    # Return result
    gg_q5
    
  })
  
# Q6: Investments by tags, date and funding ---------------------------------------------  

  # 1. Table
  output$q6_table <- DT::renderDataTable(
    DT::datatable(df_q6(), options = list(pageLength = 10, searching = TRUE), rownames= FALSE)
  )
  
  # 2. Data frame
  df_q6 <- reactive({
    # 1. get list of selected tags from input
    list_of_selected_tags <- input$q6_tags
    
    # 2. pipeline
    tags_selected <-  df_tag %>% 
      filter(name %in% list_of_selected_tags)
    
    # 3. get entities with tags
    entities_tags_selected <- df_tag_entity %>% 
      filter(tag_id %in% tags_selected$id)
    
    #write_feather(entities_tags_selected, "entities_tags_selected")
    
    # list of unique tags
    unique_tags <- unique(entities_tags_selected$tag_id)
    
    #
    # 4.1. Select entities <OR>
    #
    if (input$q6_tags_filter == "1"){
      # Selection
      entities_selected <- df_entity %>% 
        filter(id %in% entities_tags_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- entities_selected %>% 
        mutate(
          country = ifelse(country=="", "N/A", country),
          description = paste0(substr(description, 1, 20), "...")) %>% 
        subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
      
      #
      # Add info from "investment"
      #
      entities_selected <- 
        left_join(entities_selected, df_investment, by = c("id" = "entity_id")) %>% 
        subset(select = c("id", "name", "description", "country", "city", 
                          "founded_in", "employees","total_raised",
                          "amount", "currency", "investment_date")) %>% 
        mutate(investment_date = as.Date(investment_date))
      
      # Filter by "date" of investment
      entities_selected <- entities_selected %>% 
        filter((investment_date >= input$q6_dates[1]) & (investment_date <= input$q6_dates[2]))
      
      # Filter by "amount" of investment
      # 1. Change type of feature
      entities_selected <- entities_selected %>% 
        mutate(amount = as.numeric(amount))
      # 2. Selection
      if(input$q6_amount == "<     1,000,000"){
        entities_selected <- entities_selected %>% 
          filter(amount  < 1000000)
      }
      if(input$q6_amount == "      1,000,000 :    10,000,000"){
        entities_selected <- entities_selected %>% 
          filter((amount  >= 1000000) & (amount < 10000000))
      }
      if(input$q6_amount == "     10,000,000 :   100,000,000"){
        entities_selected <- entities_selected %>% 
          filter((amount  >= 10000000) & (amount < 100000000))
      }
      if(input$q6_amount == "    100,000,000 : 1,000,000,000"){
        entities_selected <- entities_selected %>% 
          filter((amount  >= 100000000) & (amount < 1000000000))
      }
      if(input$q6_amount == "> 1,000,000,000"){
        entities_selected <- entities_selected %>% 
          filter(amount  >= 1000000000)
      }

    }
    
    #
    # 4.2. Select entities <AND>
    #
    if (input$q6_tags_filter == "2"){
      
      # Select
      entities_selected <- entities_tags_selected %>% 
        group_by(entity_id) %>% 
        summarise(n = n()) %>% 
        filter(n == length(unique_tags))
      
      # Selected entities
      entities_selected_ids <- unique(entities_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- df_entity %>%
        filter(id %in% entities_selected_ids) %>%
        mutate(
          country = ifelse(country=="", "N/A", country),
          description = paste0(substr(description, 1, 20), "...")) %>% 
        subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
      
      #
      # Add info from "investment"
      #
      entities_selected <- 
        left_join(entities_selected, df_investment, by = c("id" = "entity_id")) %>% 
        subset(select = c("id", "name", "description", "country", "city", 
                          "founded_in", "employees","total_raised",
                          "amount", "currency", "investment_date")) %>% 
        mutate(investment_date = as.Date(investment_date))
      
      # Filter by "date" of investment
      entities_selected <- entities_selected %>% 
        filter((investment_date >= input$q6_dates[1]) & (investment_date <= input$q6_dates[2]))
      
      # Filter by "amount" of investment
      # 1. Change type of feature
      entities_selected <- entities_selected %>% 
        mutate(amount = as.numeric(amount))
      # 2. Selection
      if(input$q6_amount == "<     1,000,000"){
        entities_selected <- entities_selected %>% 
          filter(amount  < 1000000)
      }
      if(input$q6_amount == "      1,000,000 :    10,000,000"){
        entities_selected <- entities_selected %>% 
          filter((amount  >= 1000000) & (amount < 10000000))
      }
      if(input$q6_amount == "     10,000,000 :   100,000,000"){
        entities_selected <- entities_selected %>% 
          filter((amount  >= 10000000) & (amount < 100000000))
      }
      if(input$q6_amount == "    100,000,000 : 1,000,000,000"){
        entities_selected <- entities_selected %>% 
          filter((amount  >= 100000000) & (amount < 1000000000))
      }
      if(input$q6_amount == "> 1,000,000,000"){
        entities_selected <- entities_selected %>% 
          filter(amount  >= 1000000000)
      }
      
    }
    
    # 5. Get result
    return(entities_selected)
    
  })
  
  # Download
  output$btn_download_q6 <- downloadHandler(
    filename = "investments_by_tags_date_funding.csv",
    content = function(file) {
      write.csv(df_q6(), file)
    }
  )
  
  
    
  
}



shinyApp(ui, server)