library(shinydashboard)
library(RMySQL)
#library(sqldf)
library(feather)
library(ggplot2)
library(dplyr)
library(reshape2)
library(lubridate)
library(plotly) 
library(DT)

# 1. Data preprocessing ------------------------------------------------------

path_temp_files <- "data"

# Country code ------------------------------------------------------------

country_name <- function(code){
  data <- list(
    "AF" = "Afghanistan", "AX" = "Aland Islands", "AL" = "Albania", "DZ" = "Algeria",
    "AS" = "American Samoa", "AD" = "Andorra", "AO" = "Angola", "AI" = "Anguilla", "AQ" = "Antarctica",
    "AG" = "Antigua and Barbuda", "AR" = "Argentina", "AM" = "Armenia", "AW" = "Aruba",
    "AU" = "Australia", "AT" = "Austria", "AZ" = "Azerbaijan", "BS" = "Bahamas", "BH" = "Bahrain",
    "BD" = "Bangladesh", "BB" = "Barbados", "BY" = "Belarus", "BE" = "Belgium", "BZ" = "Belize",
    "BJ" = "Benin", "BM" = "Bermuda", "BT" = "Bhutan", "BO" = "Bolivia", "BA" = "Bosnia and Herzegovina",
    "BW" = "Botswana", "BV" = "Bouvet Island", "BR" = "Brazil", "VG" = "British Virgin Islands",
    "IO" = "British Indian Ocean Territory", "BN" = "Brunei Darussalam", "BG" = "Bulgaria",
    "BF" = "Burkina Faso", "BI" = "Burundi", "KH" = "Cambodia", "CM" = "Cameroon", "CA" = "Canada",
    "CV" = "Cape Verde", "KY" = "Cayman Islands", "CF" = "Central African Republic",
    "TD" = "Chad", "CL" = "Chile", "CN" = "China", "HK" = "Hong Kong", "MO" = "Macao",
    "CX" = "Christmas Island", "CC" = "Cocos (Keeling) Islands", "CO" = "Colombia",
    "KM" = "Comoros", "CG" = "Congo (Brazzaville)", "CD" = "Congo (Kinshasa)", "CK" = "Cook Islands",
    "CR" = "Costa Rica", "CI" = "Cote d'Ivoire", "HR" = "Croatia", "CU" = "Cuba","CY" = "Cyprus",
    "CZ" = "Czech Republic", "DK" = "Denmark", "DJ" = "Djibouti", "DM" = "Dominica",
    "DO" = "Dominican Republic", "EC" = "Ecuador", "EG" = "Egypt", "SV" = "El Salvador",
    "GQ" = "Equatorial Guinea", "ER" = "Eritrea", "EE" = "Estonia", "ET" = "Ethiopia",
    "FK" = "Falkland Islands (Malvinas)", "FO" = "Faroe Islands", "FJ" = "Fiji",
    "FI" = "Finland", "FR" = "France", "GF" = "French Guiana", "PF" = "French Polynesia",
    "TF" = "French Southern Territories", "GA" = "Gabon", "GM" = "Gambia", "GE" = "Georgia",
    "DE" = "Germany", "GH" = "Ghana", "GI" = "Gibraltar","GR" = "Greece","GL" = "Greenland",
    "GD" = "Grenada", "GP" = "Guadeloupe","GU" = "Guam","GT" = "Guatemala", "GG" = "Guernsey", "GN" = "Guinea",
    "GW" = "Guinea-Bissau", "GY" = "Guyana", "HT" = "Haiti","HM" = "Heard and Mcdonald Islands",
    "VA" = "Holy See (Vatican City State)", "HN" = "Honduras", "HU" = "Hungary", "IS" = "Iceland",
    "IN" = "India", "ID" = "Indonesia","IR" = "Iran", "IQ" = "Iraq", "IE" = "Ireland",
    "IM" = "Isle of Man", "IL" = "Israel", "IT" = "Italy", "JM" = "Jamaica", "JP" = "Japan",
    "JE" = "Jersey", "JO" = "Jordan", "KZ" = "Kazakhstan", "KE" = "Kenya", "KI" = "Kiribati",
    "KP" = "North Korea","KR" = "South Korea", "KW" = "Kuwait", "KG" = "Kyrgyzstan", "LA" = "Lao PDR",
    "LV" = "Latvia", "LB" = "Lebanon", "LS" = "Lesotho", "LR" = "Liberia", "LY" = "Libya",
    "LI" = "Liechtenstein", "LT" = "Lithuania", "LU" = "Luxembourg", "MK" = "Macedonia",
    "MG" = "Madagascar", "MW" = "Malawi", "MY" = "Malaysia", "MV" = "Maldives", "ML" = "Mali",
    "MT" = "Malta", "MH" = "Marshall Islands", "MQ" = "Martinique", "MR" = "Mauritania",
    "MU" = "Mauritius", "YT" = "Mayotte", "MX" = "Mexico", "FM" = "Micronesia", "MD" = "Moldova",
    "MC" = "Monaco", "MN" = "Mongolia", "ME" = "Montenegro", "MS" = "Montserrat", "MA" = "Morocco",
    "MZ" = "Mozambique", "MM" = "Myanmar", "NA" = "Namibia", "NR" = "Nauru","NP" = "Nepal",
    "NL" = "Netherlands", "AN" = "Netherlands Antilles", "NC" = "New Caledonia", "NZ" = "New Zealand",
    "NI" = "Nicaragua", "NE" = "Niger", "NG" = "Nigeria","NU" = "Niue","NF" = "Norfolk Island",
    "MP" = "Northern Mariana Islands","NO" = "Norway", "OM" = "Oman","PK" = "Pakistan",
    "PW" = "Palau", "PS" = "Palestinian Territory", "PA" = "Panama", "PG" = "Papua New Guinea",
    "PY" = "Paraguay", "PE" = "Peru", "PH" = "Philippines", "PN" = "Pitcairn", "PL" = "Poland",
    "PT" = "Portugal", "PR" = "Puerto Rico", "QA" = "Qatar", "RE" = "Runion", "RO" = "Romania",
    "RU" = "Russian Federation", "RW" = "Rwanda", "BL" = "Saint-Barthlemy","SH" = "Saint Helena",
    "KN" = "Saint Kitts and Nevis", "LC" = "Saint Lucia","MF" = "Saint-Martin (French part)",
    "PM" = "Saint Pierre and Miquelon","VC" = "Saint Vincent and Grenadines", "WS" = "Samoa",
    "SM" = "San Marino", "ST" = "Sao Tome and Principe", "SA" = "Saudi Arabia","SN" = "Senegal",
    "RS" = "Serbia", "SC" = "Seychelles", "SL" = "Sierra Leone", "SG" = "Singapore", "SK" = "Slovakia",
    "SI" = "Slovenia", "SB" = "Solomon Islands", "SO" = "Somalia","ZA" = "South Africa",
    "GS" = "South Georgia and the South Sandwich Islands", "SS" = "South Sudan", "ES" = "Spain",
    "LK" = "Sri Lanka", "SD" = "Sudan", "SR" = "Suriname", "SJ" = "Svalbard and Jan Mayen Islands",
    "SZ" = "Swaziland", "SE" = "Sweden", "CH" = "Switzerland", "SY" = "Syrian Arab Republic",
    "TW" = "Taiwan", "TJ" = "Tajikistan", "TZ" = "Tanzania", "TH" = "Thailand","TL" = "Timor-Leste",
    "TG" = "Togo", "TK" = "Tokelau", "TO" = "Tonga", "TT" = "Trinidad and Tobago", "TN" = "Tunisia",
    "TR" = "Turkey", "TM" = "Turkmenistan", "TC" = "Turks and Caicos Islands", "TV" = "Tuvalu",
    "UG" = "Uganda", "UK" = "United Kingdom", "UA" = "Ukraine","AE" = "United Arab Emirates",
    "GB" = "United Kingdom", "US" = "United States of America", "UM" = "US Minor Outlying Islands",
    "UY" = "Uruguay", "UZ" = "Uzbekistan", "VU" = "Vanuatu", "VE" = "Venezuela", "VN" = "Viet Nam",
    "VI" = "Virgin Islands (US)", "WF" = "Wallis and Futuna Islands", "EH" = "Western Sahara",
    "YE" = "Yemen", "ZM" = "Zambia","ZW" = "Zimbabwe")
  
  if(is.na(code) || is.null(code) || (code == "")){
    result <- "N/A"
  } else {
    result <- data[[code]]
  }
  
  return(result)
}

# 1.1. Load data ---------------------------------------------------

# Check directory for cached files
exist_cash <- function(){
  return(dir.exists(path_temp_files))
}

# Read data from file
read_df <- function(name){
  return(read_feather(paste0(path_temp_files, "/", name)))
}

# Load one table
db_table_load <- function(db, table){
  # read
  rs = dbSendQuery(db, paste0("select * from ", table))
  df = fetch(rs, n=-1)
  write_feather(df, paste0(path_temp_files, "/", table))
}


# Check cash and read/load data
# --
if (exist_cash()){
  
  # 1. Read data from cash
  # --
  
  # main tables
  df_entity <- read_df("wp_esi_entity")
  df_investment <- read_df("wp_esi_investment")
  df_technology <- read_df("wp_esi_technology")
  df_industry <- read_df("wp_esi_industry")
  df_function <- read_df("wp_esi_function")
  df_media <- read_df("wp_esi_media")
  df_region <- read_df("wp_esi_region")
  df_type <- read_df("wp_esi_type")
  df_news <- read_df("wp_esi_news")
  df_tag <- read_df("wp_esi_tag")
  
  # tags cross-tables
  df_tag_entity <- read_df("wp_esi_tag_entity")
  df_tag_news <- read_df("wp_esi_tag_news")
  
  # news cross-tables
  df_news_entity <- read_df("wp_esi_news_entity")
  
  # Packages management
  detach("package:RMySQL", unload=TRUE)
  library(sqldf)
  
} else {
  
  # 2. Load data from web
  # --
  
  library(RMySQL)
  
  # Connect db
  db <-  dbConnect(MySQL(), user='analyst', password='exa_analyst1&',
                   dbname='esi_management',
                   host='lecanaldb.c12hbxfn3xzn.eu-west-1.rds.amazonaws.com',
                   port=3306)
  
  
  # Delete directory for temp files if exists
  if(dir.exists(path_temp_files)){
    unlink(path_temp_files, recursive = TRUE, force = TRUE)  
  }
  
  # Create new directory
  dir.create(path_temp_files)
  
  # Load tables
  # main tables
  df_entity <- db_table_load(db, "wp_esi_entity")
  df_investment <- db_table_load(db, "wp_esi_investment")
  df_technology <- db_table_load(db, "wp_esi_technology")
  df_industry <- db_table_load(db, "wp_esi_industry")
  df_function <- db_table_load(db, "wp_esi_function")
  df_media <- db_table_load(db, "wp_esi_media")
  df_region <- db_table_load(db, "wp_esi_region")
  df_type <- db_table_load(db, "wp_esi_type")
  df_news <- db_table_load(db, "wp_esi_news")
  df_tag <- db_table_load(db, "wp_esi_tag")
  
  # tags cross-tables
  df_tag_entity <- db_table_load(db, "wp_esi_tag_entity")
  df_tag_news <- db_table_load(db, "wp_esi_tag_news")
  
  # news cross-tables
  df_news_entity <- db_table_load(db, "wp_esi_news_entity")
  
  # Disconnect
  dbDisconnect(db)
  
  # Packages management
  detach("package:RMySQL", unload=TRUE)
  library(sqldf)
  
}

# UI ----------------------------------------------------------------------

sidebar <- dashboardSidebar(
  sidebarMenu(
    menuItem("About", tabName = "about", icon = icon("dashboard")),
    menuItem("Companies", icon = icon("dashboard"),
             menuSubItem("Data all", tabName = "company_data_all"),
             menuSubItem("Data by tags", tabName = "company_data_tags"),
             p(),
             menuSubItem("Analysis by countries", tabName = "company_analysis_countries"),
             menuSubItem("Analysis by tags", tabName = "company_analysis_tags"),
             menuSubItem("Analysis by size per tags", tabName = "company_analysis_size_tags"),
             menuSubItem("Analysis by age per tags", tabName = "company_analysis_age_tags")
    ),
    menuItem("Investments", icon = icon("money"),
             menuSubItem("Data all", tabName = "invest_data_all"),
             menuSubItem("Data by tags", tabName = "invest_data_tags"),
             menuSubItem("Data by tags and dates", tabName = "invest_data_tags_date_fund"),
             p(),
             menuSubItem("Analysis by tags", tabName = "invest_analysis_tags")
    ),
    menuItem("News", icon = icon("file"),
             menuSubItem("Data all", tabName = "news_data_all"),
             menuSubItem("Data by tags", tabName = "news_data_tags"))
  )
)

body <- dashboardBody(
  
  # About --
  
  tabItems(
    # 1. About ----
    tabItem(tabName = "about",
            htmlOutput("system_info"),
            actionButton("btn_load_db", "Update data"),
            htmlOutput("txt_process")
    ),
    
    # 2. Companies ----
    
    tabItem(tabName = "company_data_all",
            fluidPage(
              fluidRow(h2("All data about companies")),
              fluidRow(checkboxInput("dt_company_data_all_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_company_data_all"))
            )),
    tabItem(tabName = "company_data_tags",
            fluidPage(
              fluidRow(h2("Data about companies by tags")),
              fluidRow(selectizeInput('company_data_tags_tags', 'Tags', choices = df_tag$name, multiple = TRUE)),
              fluidRow(radioButtons("company_data_tags_tags_filter", "Tags filter",
                                    choices = list("OR" = 1, "AND" = 2), selected = 1, inline = T)),
              fluidRow(checkboxInput("dt_company_data_tags_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_company_data_tags")),
              fluidRow(downloadButton('btn_download_company_data_tags', 'Download'))
            )),
    tabItem(tabName = "company_analysis_countries",
            fluidPage(
              fluidRow(h2("Analysis of companies by countries")),
              fluidRow(p(tags$b("Number of companies"))),
              fluidRow(
                column(3, textInput("company_analysis_countries_entities_min", "Min", "100")),
                column(3, textInput("company_analysis_countries_entities_max", "Max", "10000"))),
              fluidRow(plotlyOutput("company_analysis_countries")),
              fluidRow(dataTableOutput("dt_company_analysis_countries")),
              fluidRow(downloadButton('btn_download_company_analysis_countries', 'Download'))
            )
            
    ),
    tabItem(tabName = "company_analysis_tags",
            fluidPage(
              fluidRow(h2("Analysis of companies by tags")),
              fluidRow(selectInput("company_analysis_tags_choice", "Number of tags",
                                   c("> 100", "50:100", "10:50", "5:10", "3:5", "1:3", "1: Max"))),
              fluidRow(plotlyOutput("company_analysis_tags")),
              fluidRow(dataTableOutput("dt_company_analysis_tags")),
              fluidRow(downloadButton('btn_download_company_analysis_tags', 'Download'))
            )
    ),
    tabItem(tabName = "company_analysis_size_tags",
            fluidPage(
              fluidRow(h2("Analysis of companies by average size per tags")),
              fluidRow(selectInput("company_analysis_size_tags_choice", "Average size",
                                   c("> 1000", "500 : 1000", "250 : 500", "100 : 250", "50 : 100", "25 : 50", "1 : 25", "1 : Max"))),
              fluidRow(plotlyOutput("company_analysis_size_tags")),
              fluidRow(dataTableOutput("dt_company_analysis_size_tags")),
              fluidRow(downloadButton('btn_download_company_analysis_size_tags', 'Download'))
            )),
    tabItem(tabName = "company_analysis_age_tags",
            fluidPage(
              fluidRow(h2("Analysis of companies by average age per tags")),
              fluidRow(selectInput("company_analysis_age_tags_choice", "Average age",
                                   c("> 20", "15 : 20", "10 : 15", "7 : 10", "5 : 7", "3 : 5", "1 : 3", "1 : Max"))),
              fluidRow(plotlyOutput("company_analysis_age_tags")),
              fluidRow(dataTableOutput("dt_company_analysis_age_tags")),
              fluidRow(downloadButton('btn_download_company_analysis_age_tags', 'Download'))
            )),
    
    # 3. Investments ----
    
    tabItem(tabName = "invest_data_all",
            fluidPage(
              fluidRow(h2("Data about all investments")),
              fluidRow(checkboxInput("dt_invest_data_all_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_invest_data_all"))
            )
    ),
    
    tabItem(tabName = "invest_data_tags",
            fluidPage(
              fluidRow(h2("Data about investments by tags")),
              fluidRow(selectizeInput('invest_data_tags_tags', 'Tags', choices = df_tag$name, multiple = TRUE)),
              fluidRow(radioButtons("invest_data_tags_tags_filter", "Tags filter",
                                    choices = list("OR" = 1, "AND" = 2), selected = 1, inline = T)),
              fluidRow(checkboxInput("dt_invest_data_tags_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_invest_data_tags")),
              fluidRow(downloadButton('btn_download_invest_data_tags', 'Download'))
            )),
    
    
    tabItem(tabName = "invest_analysis_tags",
            fluidPage(
              fluidRow(h2("Analysis of investments per tags")),
              fluidRow(uiOutput('invest_analysis_tags_slider')),
              fluidRow(plotlyOutput("invest_analysis_tags")),
              fluidRow(dataTableOutput("dt_invest_analysis_tags")),
              fluidRow(downloadButton('btn_download_invest_analysis_tags', 'Download'))
            )
            
    ),
    
    tabItem(tabName = "invest_data_tags_date_fund",
            fluidPage(
              fluidRow(h2("Data about investments by tags and dates")),
              fluidRow(selectizeInput('invest_data_tags_date_fund_tags', 'Tags', choices = df_tag$name, multiple = TRUE)),
              fluidRow(radioButtons("invest_data_tags_date_fund_tags_filter", "Tags filter",
                                    choices = list("OR" = 1, "AND" = 2), selected = 1, inline = T)),
              fluidRow(
                dateRangeInput("invest_data_tags_date_fund_dates", label = h3("Date range"),
                               start = min(as.Date(df_investment$investment_date)), 
                               end = max(as.Date(df_investment$investment_date)))),
              fluidRow(checkboxInput("dt_invest_data_tags_date_fund_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_invest_data_tags_date_fund")),
              fluidRow(downloadButton('btn_download_invest_data_tags_date_fund', 'Download'))
            )),
    # 4. News ----
    tabItem(tabName = "news_data_all",
            fluidPage(
              fluidRow(h2("Data about all news")),
              fluidRow(checkboxInput("dt_news_data_all_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_news_data_all"))
            )),
    
    tabItem(tabName = "news_data_tags",
            fluidPage(
              fluidRow(h2("Data about news by tags")),
              fluidRow(selectizeInput('news_data_tags_tags', 'Tags', choices = df_tag$name, multiple = TRUE)),
              fluidRow(radioButtons("news_data_tags_tags_filter", "Tags filter",
                                    choices = list("OR" = 1, "AND" = 2), selected = 1, inline = T)),
              fluidRow(checkboxInput("dt_news_data_tags_filter", "Filter", FALSE)),
              fluidRow(dataTableOutput("dt_news_data_tags")),
              fluidRow(downloadButton('btn_download_news_data_tags', 'Download'))
            ))
    
  )
)


ui <- dashboardPage(
  dashboardHeader(title = "ESI Analytics"),
  sidebar,
  body
)

# SERVER ------------------------------------------------------------------

server <- function(input, output, session) {
  
  # 1. About info -------------------------------------------------------------  
  
  output$system_info <- renderUI({
    HTML(paste0(
      "<p><b>General information</b>: </b></p>",
      "<p>Entities: <b>", nrow(df_entity), "</b></p>",
      "<p>Tags: <b>", nrow(df_tag), "</b></p>"
    ))
    
  })
  
  # Load
  observeEvent(input$btn_load_db, {
    # 1. do it
    load_web_db()
    # todo: remove copy/paste
    df_entity <- read_df("wp_esi_entity")
    df_tag <- read_df("wp_esi_tag")
    df_tag_entity <- read_df("wp_esi_tag_entity")
    df_investment <- read_df("wp_esi_investment")
    
    # 3. show message
    output$txt_process <- renderUI({
      HTML("Database updated")
    }) 
    
  })
  
  
  # 2. COMPANIES --------------------------------------------------------------
  
  
  # 2.1. company_data_all ---------------------------------------------
  
  # 1. Data frame
  df_company_data_all <- reactive({
    df_entity %>% 
      mutate(
        country = as.character(lapply(country, country_name)),
        url = paste0("<a href='", url,"' target='_blank'>", url,"</a>")) %>% 
      subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
  })
  
  # 2. Table
  output$dt_company_data_all <- renderDataTable(
    if(input$dt_company_data_all_filter){
      datatable(df_company_data_all(), options = list(
        pageLength = 25,
        searchHighlight = TRUE,
        columnDefs = list(list(
          targets = 4,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = "top")
    } else {
      datatable(df_company_data_all(), options = list(
        pageLength = 25,
        searchHighlight = TRUE,
        columnDefs = list(list(
          targets = 4,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = "none")
    }
  )

  # 2.2. company_data_tags ---------------------------------------------
  
  # 1. Table
  output$dt_company_data_tags <- renderDataTable(
    if(input$dt_company_data_tags_filter){
      datatable(df_company_data_tags(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 4,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = 'top')
    } else {
      datatable(df_company_data_tags(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 4,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = 'none')
    }
    
    # datatable(df_company_data_tags(), options = list(pageLength = 10, searching = TRUE), rownames= FALSE)
  )
  
  # 2. Data frame
  df_company_data_tags <- reactive({
    
    # 1. get list of selected tags from input
    list_of_selected_tags <- input$company_data_tags_tags
    
    # 2. pipeline
    tags_selected <-  df_tag %>% 
      filter(name %in% list_of_selected_tags)
    
    # 3. get entities with tags
    entities_tags_selected <- df_tag_entity %>% 
      filter(tag_id %in% tags_selected$id)
    
    #write_feather(entities_tags_selected, "entities_tags_selected")
    
    # list of unique tags
    unique_tags <- unique(entities_tags_selected$tag_id)
    
    #
    # Check type of filter
    #
    # 4.1. Select entities <OR>
    if (input$company_data_tags_tags_filter == "1"){
      # Selection
      entities_selected <- df_entity %>% 
        filter(id %in% entities_tags_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- entities_selected %>% 
        mutate(
          country = as.character(lapply(country, country_name)),
          url = paste0("<a href='", url,"' target='_blank'>", url,"</a>")) %>% 
        subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
      
    }
    
    # 4.2. Select entities <AND>
    if (input$company_data_tags_tags_filter == "2"){
      
      # Select
      entities_selected <- entities_tags_selected %>% 
        group_by(entity_id) %>% 
        summarise(n = n()) %>% 
        filter(n == length(unique_tags))
      
      # Selected entities
      entities_selected_ids <- unique(entities_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- df_entity %>%
        filter(id %in% entities_selected_ids) %>%
        mutate(
          country = as.character(lapply(country, country_name)),
          url = paste0("<a href='", url,"' target='_blank'>", url,"</a>")) %>% 
        subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
      
    }
    
    # 5. Get result
    return(entities_selected)
    
  })
  
  # Download
  output$btn_download_company_data_tags <- downloadHandler(
    filename = "companies_by_tags.csv",
    content = function(file) {
      write.csv(df_company_data_tags(), file)
    }
  )
  
  # 2.3. company_analysis_countries --------------------------
  
  # 1. Range of companies from min to max 
  company_analysis_countries_get_min <- renderText({
    input$company_analysis_countries_entities_min 
  })
  
  company_analysis_countries_get_max <- renderText({ 
    input$company_analysis_countries_entities_max 
  })
  
  # 1. Data frame
  df_company_analysis_countries <- reactive({
    df_entity %>% 
      group_by(country) %>% 
      summarise(sum = n()) %>% 
      arrange(-sum) %>% 
      mutate(country = as.character(lapply(country, country_name)))  
  })
  
  # 2. Table
  output$dt_company_analysis_countries <- renderDataTable(
    datatable(df_company_analysis_countries(),  options = list(pageLength = 10)) # , searching = FALSE
  )
  
  # 3. Graph
  output$company_analysis_countries <- renderPlotly({
    
    gg_title <- paste0("Countries with ", 
                       company_analysis_countries_get_min(), " : ", 
                       company_analysis_countries_get_max(),  " companies")
    
    gg_company_analysis_countries <- df_company_analysis_countries() %>% 
      filter(sum %in% company_analysis_countries_get_min():company_analysis_countries_get_max()) %>% 
      ggplot(aes(reorder(country, sum), sum, fill=factor(sum))) +
      ggtitle(gg_title) +
      xlab("Countries") +
      ylab("Number of companies") +  
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()
    
    # Return result
    gg_company_analysis_countries
    
  })
  
  # Download
  output$btn_download_company_analysis_countries <- downloadHandler(
    filename = "company_analysis_by_countries.csv",
    content = function(file) {
      write.csv(df_company_analysis_countries(), file)
    }
  )
  
  # 2.4. company_analysis_tags --------------------------
  
  # 1. Data frame
  df_company_analysis_tags <- reactive({
    # Q3.1. Join data
    s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
    s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
    x <- sqldf(s1)
    x <- sqldf(s2)
    
    colnames(x)
    # Q3.2. Change column names
    colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                     "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                     "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                     "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                     "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
    
    # Q3.3. Select features
    x <- x %>% 
      subset(select = c("entity_id", "total_raised", "tag_id", "tag_name"))
    
    # Q3.4. Transform data set
    xt <- dcast(x, entity_id ~ tag_id)
    
    xt[!is.na(xt)] <- 1
    
    # change column types
    xt[ , colnames(xt)] <-  apply(xt[ , colnames(xt)], 2, function(x) as.numeric(as.character(x)))
    
    # calculate sums
    xt["total", ] <- colSums(xt, na.rm = T)
    xt <- xt["total", ]
    
    # remove feature
    xt$entity_id <- NULL
    
    # transpose
    xt <- t(xt)
    
    # add "tag_id"
    rnames <- as.data.frame(row.names(xt))
    colnames(rnames) <- c("tag_id")
    
    # merge
    xt <- cbind(rnames, xt)
    
    # change row names
    row.names(xt) <- NULL
    
    # add tag names
    xt <- sqldf("select xt.*, df_tag.* from xt left join df_tag on xt.tag_id = df_tag.id")
    
    # select features and arrange
    xt <- xt %>% 
      subset(select = c("name", "total")) %>% 
      arrange(-total)
    
    # get result
    return(xt)
    
  })
  
  # 2. Table
  output$dt_company_analysis_tags <- renderDataTable(
    datatable(df_company_analysis_tags(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_company_analysis_tags <- downloadHandler(
    filename = "companies_analysis_by_tags.csv",
    content = function(file) {
      write.csv(df_company_analysis_tags(), file)
    }
  )
  
  # Draw plot
  output$company_analysis_tags <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$dt_company_analysis_tags_rows_selected
    if(length(selected_rows) > 0){
      df_company_analysis_tags_plot <- df_company_analysis_tags()[selected_rows, ]
    } else {
      df_company_analysis_tags_plot <- df_company_analysis_tags()  
    }
    
    #
    # Q2.13. Draw graphs
    #
    
    # Select graph
    
    # 1:max companies    
    if(input$company_analysis_tags_choice == "1: Max"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total %in% 1:max(df_company_analysis_tags_plot$total)) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (1 : Max)")
    }
    
    # > 100 companies    
    if(input$company_analysis_tags_choice == "> 100"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total > 100) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (> 100)")
    }
    
    # 50:100
    if(input$company_analysis_tags_choice == "50:100"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total %in% 50:100) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (50-100)")
    }
    
    # 10:50
    if(input$company_analysis_tags_choice == "10:50"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total %in% 10:50) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (10-50)")
    }
    
    
    # 1:10
    if(input$company_analysis_tags_choice == "5:10"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total %in% 5:10) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (5-10)")
    }
    
    # 3:5
    if(input$company_analysis_tags_choice == "3:5"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total %in% 3:5) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (3-5)")
    }
    
    # 1:3
    if(input$company_analysis_tags_choice == "1:3"){
      gg_company_analysis_tags <- df_company_analysis_tags_plot %>% 
        filter(total %in% 1:3) %>% 
        ggplot(aes(reorder(name, total), total, fill=factor(total))) +
        ggtitle("Companies per Tag (1-3)")
    }
    
    
    # the same for all graphs
    gg_company_analysis_tags <- gg_company_analysis_tags +
      xlab("") +
      ylab("Number of companies") +  
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      theme_bw() +
      coord_flip()
    
    # Return result
    gg_company_analysis_tags
    
  })
  
  # 2.5. company_analysis_size_tags --------------------------  
  df_company_analysis_size_tags <- reactive({
    # Q4.1. Join data
    s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
    s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
    x <- sqldf(s1)
    x <- sqldf(s2)
    
    # Q4.2. Change column names
    colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                     "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                     "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                     "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                     "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
    
    # Q4.3. Select features
    x <- x %>% 
      subset(select = c("entity_id", "employees", "tag_id")) %>% 
      filter(!is.na(tag_id))
    
    # Q3.4. Transform data set
    xt <- dcast(x, tag_id ~ employees)
    
    # Q3.6. Create tags column
    xt_tags <- xt %>% 
      subset(select = c("tag_id"))
    
    #
    # Q3.7. Create "sum entities"
    #
    xt_sum_entities <- xt %>% 
      subset(select=2:length(colnames(xt))) %>% 
      mutate(sum_entities = rowSums(.)) %>% 
      subset(select = c("sum_entities"))
    
    #
    # Q3.8. Create "sum employees"
    #
    
    # For one tag function
    one_tag <- function(id){
      # 1. basic data set
      xt_case <- xt %>% 
        filter(tag_id == id) %>%
        subset(select=2:length(colnames(xt))) %>% 
        t()
      
      # 2. add "employees"
      rnames <- as.data.frame(row.names(xt_case))
      colnames(rnames) <- c("employees")
      
      xt_case <- cbind(rnames, xt_case)
      row.names(xt_case) <- NULL
      
      colnames(xt_case) <- c("employees", "sum_entities")
      
      xt_case <- xt_case %>% 
        mutate(employees = as.numeric(as.character(xt_case$employees)),
               sum_employees = employees * sum_entities) %>% 
        subset(select = c("sum_employees"))
      
      return(xt_case)
    }
    
    # init result data set
    z <- one_tag(unique(xt_tags$tag_id)[1])  
    colnames(z) <- c(unique(xt_tags$tag_id)[1])
    
    # for loop
    end_for <- unique(xt_tags$tag_id)[2:length(unique(xt_tags$tag_id))]
    for(i in end_for){
      tmp <- one_tag(i)
      colnames(tmp) <- c(i)
      z <- cbind(z, tmp)
    }
    
    # transpose
    z <- t(z)
    
    # 1. sum employees
    xt_sum_employees <- as.data.frame(z) %>% 
      mutate(sum_employees = rowSums(.)) %>% 
      subset(select = c("sum_employees"))
    
    
    # Q3.9. Merge data sets
    xt_res <- cbind(xt_tags, xt_sum_entities)
    xt_res <- cbind(xt_res, xt_sum_employees)
    
    # Q3.10. Calculate average size
    xt_res <- xt_res %>%  
      mutate(sum_entities = as.numeric(sum_entities),
             sum_employees = as.numeric(sum_employees),
             avr_size = round(sum_employees / sum_entities)) %>% 
      subset(select = c("tag_id", "avr_size"))
    
    # Q3.10. Add tag names
    
    s <- "select xt_res.*, df_tag.* from xt_res left join df_tag on xt_res.tag_id = df_tag.id"
    df_res_final <- sqldf(s)
    
    df_res_final <- df_res_final %>% 
      subset(select = c("tag_id", "name", "avr_size"))
    
    # get result
    return(df_res_final)
    
  })
  
  # 2. Table
  output$dt_company_analysis_size_tags <- renderDataTable(
    datatable(df_company_analysis_size_tags(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_company_analysis_size_tags <- downloadHandler(
    filename = "companies_average_size_per_tags.csv",
    content = function(file) {
      write.csv(df_company_analysis_size_tags(), file)
    }
  )
  
  # Draw plot
  output$company_analysis_size_tags <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$dt_company_analysis_size_tags_rows_selected
    if(length(selected_rows) > 0){
      df_company_analysis_size_tags_plot <- df_company_analysis_size_tags()[selected_rows, ]
    } else {
      df_company_analysis_size_tags_plot <- df_company_analysis_size_tags()  
    }
    
    # 1:max companies    
    if(input$company_analysis_size_tags_choice == "1 : Max"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>%
        filter(avr_size %in% 1:max(df_company_analysis_size_tags_plot$avr_size)) %>%
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per tags (1 : Max)")
    }
    
    # > 1000
    if(input$company_analysis_size_tags_choice == "> 1000"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size > 1000) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (> 1000)")
    }
    
    # 500 : 1000
    if(input$company_analysis_size_tags_choice == "500 : 1000"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size %in% 500:1000) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (500 : 1000)")
    }
    
    # 250 : 500
    if(input$company_analysis_size_tags_choice == "250 : 500"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size %in% 250:500) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (250 : 500)")
    }
    
    # 100 : 250
    if(input$company_analysis_size_tags_choice == "100 : 250"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size %in% 100:250) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (100 : 250)")
    }
    
    # 50 : 100
    if(input$company_analysis_size_tags_choice == "50 : 100"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size %in% 50:100) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (50 : 100)")
    }    
    
    # 25 : 50
    if(input$company_analysis_size_tags_choice == "25 : 50"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size %in% 25:50) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (25 : 50)")
    }    
    
    # 1 : 25
    if(input$company_analysis_size_tags_choice == "1 : 25"){
      gg_company_analysis_size_tags <- df_company_analysis_size_tags_plot %>% 
        filter(avr_size %in% 1:25) %>% 
        ggplot(aes(reorder(name, avr_size), avr_size, fill=factor(avr_size))) +
        ggtitle("Average size of companies per Tag (1 : 25)")
    }    
    
    # the same for all graphs
    gg_company_analysis_size_tags <- gg_company_analysis_size_tags +
      xlab("") +
      ylab("Average number of employees") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      theme_bw() +
      coord_flip()
    
    # Return result
    gg_company_analysis_size_tags
    
  })
  
  # 2.6. company_analysis_age_tags --------------------------  
  
  df_company_analysis_age_tags <- reactive({
    # Q5.1. Join data
    s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
    s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
    x <- sqldf(s1)
    x <- sqldf(s2)
    
    # Q5.2. Change column names
    colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                     "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                     "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                     "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                     "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
    
    # Q5.3. Check if a value is the date
    is_date <- function(mydate, date.format = "%Y-%m-%d") {
      tryCatch(!is.na(as.Date(mydate, date.format)),  
               error = function(err) {FALSE})  
    }
    
    # fix data bugs
    # x <- x %>% 
    #   filter(! is_date(founded_in)) %>% 
    #   subset(select = c("entity_id", "name", "founded_in")) %>% 
    #   unique() %>% 
    #   write.csv("bugs_founded_in.csv", row.names=F)
    
    # Q5.4. Select features
    # year > 1900
    x <- x %>% 
      subset(select = c("entity_id", "founded_in", "tag_id")) %>% 
      filter(!is.na(tag_id) & is_date(founded_in) & (year(as.Date(founded_in))>1900)) %>% 
      mutate(age = interval(as.Date(founded_in), today()) / years(1)) %>% 
      mutate(age = round(age, 2)) %>% 
      subset(select = c("entity_id", "age", "tag_id"))
    
    # Q5.5. Transform data set
    xt <- dcast(x, tag_id ~ age)
    
    # Q5.6. Create tags column
    xt_tags <- xt %>% 
      subset(select = c("tag_id"))
    
    #
    # Q5.7. Create "sum entities"
    #
    xt_sum_entities <- xt %>% 
      subset(select=2:length(colnames(xt))) %>% 
      mutate(sum_entities = rowSums(.)) %>% 
      subset(select = c("sum_entities"))
    
    #
    # Q5.8. Create "sum age" # instead 'employees'
    #
    
    # For one tag function
    one_tag <- function(id){
      # 1. basic data set
      xt_case <- xt %>% 
        filter(tag_id == id) %>%
        subset(select=2:length(colnames(xt))) %>% 
        t()
      
      # 2. add "age"
      rnames <- as.data.frame(row.names(xt_case))
      colnames(rnames) <- c("age")
      
      xt_case <- cbind(rnames, xt_case)
      row.names(xt_case) <- NULL
      
      colnames(xt_case) <- c("age", "sum_entities")
      
      xt_case <- xt_case %>% 
        mutate(age = as.numeric(as.character(xt_case$age)),
               sum_age = age * sum_entities) %>% 
        subset(select = c("sum_age"))
      
      return(xt_case)
    }
    
    # init result data set
    z <- one_tag(unique(xt_tags$tag_id)[1])  
    colnames(z) <- c(unique(xt_tags$tag_id)[1])
    
    # for loop
    end_for <- unique(xt_tags$tag_id)[2:length(unique(xt_tags$tag_id))]
    for(i in end_for){
      tmp <- one_tag(i)
      colnames(tmp) <- c(i)
      z <- cbind(z, tmp)
    }
    
    # transpose
    z <- t(z)
    
    # 1. sum age
    xt_sum_age <- as.data.frame(z) %>% 
      mutate(sum_age = rowSums(.)) %>% 
      subset(select = c("sum_age"))
    
    
    # Q3.9. Merge data sets
    xt_res <- cbind(xt_tags, xt_sum_entities)
    xt_res <- cbind(xt_res, xt_sum_age)
    
    # Q3.10. Calculate average age
    xt_res <- xt_res %>%  
      mutate(sum_entities = as.numeric(sum_entities),
             sum_age = as.numeric(sum_age),
             avr_age = round(sum_age / sum_entities), 2) %>% 
      subset(select = c("tag_id", "avr_age"))
    
    # Q3.10. Add tag names
    
    s <- "select xt_res.*, df_tag.* from xt_res left join df_tag on xt_res.tag_id = df_tag.id"
    df_res_final <- sqldf(s)
    
    df_res_final <- df_res_final %>% 
      subset(select = c("tag_id", "name", "avr_age"))
    
    # get result
    return(df_res_final)
    
  })
  
  # 2. Table
  output$dt_company_analysis_age_tags <- renderDataTable(
    datatable(df_company_analysis_age_tags(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_company_analysis_age_tags <- downloadHandler(
    filename = "companies_average_age_by_tags.csv",
    content = function(file) {
      write.csv(df_company_analysis_age_tags(), file)
    }
  )
  
  # Draw plot
  output$company_analysis_age_tags <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$dt_company_analysis_age_tags_rows_selected
    if(length(selected_rows) > 0){
      df_company_analysis_age_tags_plot <- df_company_analysis_age_tags()[selected_rows, ]
    } else {
      df_company_analysis_age_tags_plot <- df_company_analysis_age_tags()  
    }
    
    # 1:max companies    
    if(input$company_analysis_age_tags_choice == "1 : Max"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>%
        filter(avr_age %in% 1:max(df_company_analysis_age_tags_plot$avr_age)) %>%
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (1 : Max)")
    }
    
    # > 20
    if(input$company_analysis_age_tags_choice == "> 20"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age > 20) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (> 20)")
    }
    
    # 15 : 20
    if(input$company_analysis_age_tags_choice == "15 : 20"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age %in% 15:20) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (15 : 20)")
    }
    
    # 10 : 15
    if(input$company_analysis_age_tags_choice == "10 : 15"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age %in% 10:15) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (10 : 15)")
    }
    
    # 7 : 10
    if(input$company_analysis_age_tags_choice == "7 : 10"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age %in% 7:10) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (7 : 10)")
    }
    
    # 5 : 7
    if(input$company_analysis_age_tags_choice == "5 : 7"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age %in% 5:7) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (5 : 7)")
    }
    
    # 3 : 5
    if(input$company_analysis_age_tags_choice == "3 : 5"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age %in% 3:5) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (3 : 5)")
    }
    
    # 1 : 3
    if(input$company_analysis_age_tags_choice == "1 : 3"){
      gg_company_analysis_age_tags <- df_company_analysis_age_tags_plot %>% 
        filter(avr_age %in% 1:3) %>% 
        ggplot(aes(reorder(name, avr_age), avr_age, fill=factor(avr_age))) +
        ggtitle("Average age of companies per tags (1 : 3)")
    }
    
    # the same for all graphs
    gg_company_analysis_age_tags <- gg_company_analysis_age_tags +
      xlab("") +
      ylab("Average age of companies") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      theme_bw() +
      coord_flip()
    
    # Return result
    gg_company_analysis_age_tags
    
  })
  
  
  #
  # 3. INVESTMENTS ---------------------------------------------
  #
  
  # 3.1. invest_data_all ---------------------------------------------
  
  # 1. Table
  output$dt_invest_data_all <- renderDataTable(
    if(input$dt_invest_data_all_filter){
      datatable(df_invest_data_all(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 3,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        filter = 'top')
    } else {
      datatable(df_invest_data_all(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 3,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        filter = 'none')
    }
    
    
    # datatable(df_invest_data_all(), options = list(pageLength = 25), rownames= FALSE)
  )
  
  # 2. Data frame
  df_invest_data_all <- reactive({
    
    # 1. Select Entities
    df_result <- df_entity %>% 
      mutate(
        country = as.character(lapply(country, country_name))) %>% 
      subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
    
    
    # 2. Add info from "investment"
    df_result <- 
      left_join(df_result, df_investment, by = c("id" = "entity_id")) %>% 
      subset(select = c("id", "name", "description", "country", "city", 
                        "founded_in", "employees","total_raised",
                        "amount", "currency", "investment_date")) %>% 
      mutate(
        investment_date = as.Date(investment_date),
        amount = as.numeric(amount))
    
    # 3. Return result
    return(df_result)
    
  })
  
  
  # 3.2. invest_data_tags ---------------------------------------------
  
  # 1. Table
  output$dt_invest_data_tags <- renderDataTable(
    if(input$dt_invest_data_tags_filter){
      datatable(df_invest_data_tags(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 3,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        filter = 'top')
    } else {
      datatable(df_invest_data_tags(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 3,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        filter = 'none')
    }
    
    # datatable(df_invest_data_tags(), options = list(pageLength = 10, searching = TRUE), rownames= FALSE)
  )
  
  # 2. Data frame
  df_invest_data_tags <- reactive({
    
    # 1. get list of selected tags from input
    list_of_selected_tags <- input$invest_data_tags_tags
    
    # 2. pipeline
    tags_selected <-  df_tag %>% 
      filter(name %in% list_of_selected_tags)
    
    
    # return(df_invest_data_all())
    
    # 3. get entities with tags
    entities_tags_selected <- df_tag_entity %>%
      filter(tag_id %in% tags_selected$id)
    
    #write_feather(entities_tags_selected, "entities_tags_selected")
    
    # list of unique tags
    unique_tags <- unique(entities_tags_selected$tag_id)
    
    #
    # Check type of filter
    #
    # 4.1. Select entities <OR>
    if (input$invest_data_tags_tags_filter == "1"){
      # Selection
      entities_selected <- df_entity %>%
        filter(id %in% entities_tags_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- entities_selected %>%
        mutate(
          country = as.character(lapply(country, country_name))) %>%
        subset(select = c(name, url, description, country, city, founded_in, employees, total_raised))
    }
    
    # 4.2. Select entities <AND>
    if (input$invest_data_tags_tags_filter == "2"){
      
      # Select
      entities_selected <- entities_tags_selected %>% 
        group_by(entity_id) %>% 
        summarise(n = n()) %>% 
        filter(n == length(unique_tags))
      
      # Selected entities
      entities_selected_ids <- unique(entities_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- df_entity %>%
        filter(id %in% entities_selected_ids) %>%
        mutate(country = as.character(lapply(country, country_name))) %>% 
        subset(select = c(name, url, description, country, city, founded_in, employees, total_raised))
    }
    
    # 5. Get result
    return(entities_selected)
    
  })
  
  # 3. Download
  output$btn_download_invest_data_tags <- downloadHandler(
    filename = "invest_by_tags.csv",
    content = function(file) {
      write.csv(df_invest_data_tags(), file)
    }
  )
  
  
  # 3.3. invest_analysis_tags --------------------------
  
  # 1. Data frame
  df_invest_analysis_tags <- reactive({
    
    # data file for Q2 question
    invest_analysis_tags_data_file <- paste0(path_temp_files, "/invest_analysis_tags_data")
    
    if(file.exists(invest_analysis_tags_data_file)){
      df_res_final <- read_feather(invest_analysis_tags_data_file)
    }  else {
      
      # Q2.1. Join data
      s1 <- "select df_entity.*, df_tag_entity.* from df_entity left join df_tag_entity on df_entity.id=df_tag_entity.entity_id"
      s2 <- "select x.*, df_tag.* from x left join df_tag on x.tag_id = df_tag.id"
      x <- sqldf(s1)
      
      x <- sqldf(s2)
      
      # Q2.2. Change column names
      colnames(x) <- c("entity_id", "name", "description", "url", "is_investor", "country", 
                       "city", "entity_type_id", "founded_in", "employees", "acquired_by", 
                       "acquired_at", "acquired_amount", "total_raised", "latest_round_type", 
                       "facebook_link", "twitter_link", "linkedin_link", "status", "tag_id", 
                       "entity_id_1", "tag_id_1", "tag_name", "description_1", "parent_1")
      
      # Q2.3. Select features
      x <- x %>% 
        subset(select = c("entity_id", "total_raised", "tag_id")) %>% 
        filter(!is.na(tag_id))
      
      # Q2.4. Transform data set
      xt <- dcast(x, tag_id ~ total_raised)
      
      # Q2.5. Create tags column
      xt_tags <- xt %>% 
        subset(select = c("tag_id"))
      
      # Q2.6. For one tag function
      one_tag <- function(id){
        # 1. basic data set
        xt_case <- xt %>% 
          filter(tag_id == id) %>%
          subset(select=2:length(colnames(xt))) %>% 
          t()
        
        # 2. add "total_raised"
        rnames <- as.data.frame(row.names(xt_case))
        colnames(rnames) <- c("total_raised")
        
        xt_case <- cbind(rnames, xt_case)
        row.names(xt_case) <- NULL
        
        colnames(xt_case) <- c("total_raised", "sum_entities")
        
        xt_case <- xt_case %>% 
          mutate(total_raised = as.numeric(as.character(total_raised))) %>% 
          filter(!is.na(total_raised)) %>% 
          mutate(sum_total_raised = total_raised * sum_entities) %>% 
          subset(select = c("sum_total_raised"))
        
        return(xt_case)
      } # one_tag
      
      # Q2.7. init result data set
      z <- one_tag(unique(xt_tags$tag_id)[1])  
      colnames(z) <- c(unique(xt_tags$tag_id)[1])
      
      # 1. Progress object
      progress_invest_analysis_tags <- shiny::Progress$new()
      on.exit(progress_invest_analysis_tags$close())
      progress_invest_analysis_tags$set(message = "Processing", value = 0)
      
      # Q2.8. for loop
      end_for <- unique(xt_tags$tag_id)[2:length(unique(xt_tags$tag_id))]
      for(i in end_for){
        # Show progress
        progress_invest_analysis_tags$inc(1/length(end_for), detail = paste("", i))
        
        tmp <- one_tag(i)
        colnames(tmp) <- c(i)
        z <- cbind(z, tmp)
      }
      
      # Q2.9. transpose
      z <- t(z)
      
      # Q2.10. sum total_raised
      xt_sum_total_raised <- as.data.frame(z) %>% 
        mutate(sum_total_raised = rowSums(.)) %>% 
        subset(select = c("sum_total_raised"))
      
      # Q2.11. Merge data sets
      xt_res <- cbind(xt_tags, xt_sum_total_raised)
      
      # Q2.12. Add tag names
      s <- "select xt_res.*, df_tag.* from xt_res left join df_tag on xt_res.tag_id = df_tag.id"
      df_res_final <- sqldf(s)
      
      df_res_final <- df_res_final %>% 
        subset(select = c("tag_id", "name", "sum_total_raised"))
      
      # Cash files
      write_feather(df_res_final, invest_analysis_tags_data_file)
      
    }    
    
    # get result
    return(df_res_final)
    
  })
  
  # Modify "df_q2" by converting "sum_total_raised" into 1,000 "K" scale
  df_invest_analysis_tags_modif <- reactive({
    return(df_invest_analysis_tags() %>% 
             mutate(sum_total_raised = round(sum_total_raised / 1000, 3)))
  })
  
  # Slider for "Total raised"
  output$invest_analysis_tags_slider <- renderUI({
    max_total_raised <- max(df_invest_analysis_tags_modif()$sum_total_raised)
    sliderInput("invest_analysis_tags_slider", "Total raised (1K scale)", 
                min = 0, max = max_total_raised, 
                value = c(3000000, max_total_raised) )
  })
  
  # 2. Table
  output$dt_invest_analysis_tags <- renderDataTable(
    datatable(df_invest_analysis_tags_modif(), options = list(pageLength = 25))
  )
  
  # Download
  output$btn_download_invest_analysis_tags <- downloadHandler(
    filename = "investment_total_raised_by_tags.csv",
    content = function(file) {
      write.csv(df_invest_analysis_tags_modif(), file)
    }
  )
  
  # Draw plot
  output$invest_analysis_tags <- renderPlotly({
    
    # Check selected rows
    selected_rows <- input$dt_invest_analysis_tags_rows_selected
    if(length(selected_rows) > 0){
      df_invest_analysis_tags_plot <- df_invest_analysis_tags_modif()[selected_rows, ]
    } else {
      df_invest_analysis_tags_plot <- df_invest_analysis_tags_modif()  
    }
    
    #
    # Q2.13. Draw graphs
    #
    
    min_total_raised <- input$invest_analysis_tags_slider[1]
    max_total_raised <- input$invest_analysis_tags_slider[2]
    
    gtitle <- paste0("Total raised of companies per Tag (",min_total_raised, " : ", max_total_raised, ")")
    
    g <- df_invest_analysis_tags_plot %>%
      filter((sum_total_raised >= min_total_raised) & (sum_total_raised <= max_total_raised)) %>%
      ggplot(aes(reorder(name, sum_total_raised), sum_total_raised, fill=factor(sum_total_raised))) +
      ggtitle(gtitle) +
      xlab("") +
      ylab("Total raised of company") +
      geom_bar(stat="identity") +
      scale_fill_discrete(name = "Number") +
      coord_flip()
    
    # Return result
    g
    
  })
  
  
  # 3.4. invest_data_tags_date_fund ---------------------------------------------  


  # 1. Table
  output$dt_invest_data_tags_date_fund <- renderDataTable(
    if(input$dt_invest_data_tags_date_fund_filter){
      datatable(df_invest_data_tags_date_fund(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 3,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        filter = 'top')
    } else {
      datatable(df_invest_data_tags_date_fund(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 3,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 6 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 6) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        filter = 'none')
    }
    
    # datatable(df_invest_data_tags_date_fund_modif(), options = list(pageLength = 10, searching = TRUE), rownames= FALSE)
  )
  
  # Download
  output$btn_download_invest_data_tags_date_fund <- downloadHandler(
    filename = "investments_by_tags_date_funding.csv",
    content = function(file) {
      write.csv(df_invest_data_tags_date_fund, file)
    }
  )
  
  
  # 2. Data frame
  df_invest_data_tags_date_fund <- reactive({
    # 1. get list of selected tags from input
    list_of_selected_tags <- input$invest_data_tags_date_fund_tags
    
    # 2. pipeline
    tags_selected <-  df_tag %>% 
      filter(name %in% list_of_selected_tags)
    
    # 3. get entities with tags
    entities_tags_selected <- df_tag_entity %>% 
      filter(tag_id %in% tags_selected$id)
    
    #write_feather(entities_tags_selected, "entities_tags_selected")
    
    # list of unique tags
    unique_tags <- unique(entities_tags_selected$tag_id)
    
    #
    # 4.1. Select entities <OR>
    #
    if (input$invest_data_tags_date_fund_tags_filter == "1"){
      # Selection
      entities_selected <- df_entity %>% 
        filter(id %in% entities_tags_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- entities_selected %>% 
        mutate(country = as.character(lapply(country, country_name))) %>% 
        subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
      
      #
      # Add info from "investment"
      #
      entities_selected <- 
        left_join(entities_selected, df_investment, by = c("id" = "entity_id")) %>% 
        subset(select = c("id", "name", "description", "country", "city", 
                          "founded_in", "employees","total_raised",
                          "amount", "currency", "investment_date")) %>% 
        mutate(investment_date = as.Date(investment_date))
      
      # Filter
      entities_selected <- entities_selected %>% 
        # 1. by "date" of investment
        filter((investment_date >= input$invest_data_tags_date_fund_dates[1]) & (investment_date <= input$invest_data_tags_date_fund_dates[2])) %>% 
        # 2. change type and scale to 1,000 (1K)
        #mutate(amount = round(as.numeric(amount)/1000, 3) )
        mutate(amount = as.numeric(amount))
    }
    
    #
    # 4.2. Select entities <AND>
    #
    if (input$invest_data_tags_date_fund_tags_filter == "2"){
      
      # Select
      entities_selected <- entities_tags_selected %>% 
        group_by(entity_id) %>% 
        summarise(n = n()) %>% 
        filter(n == length(unique_tags))
      
      # Selected entities
      entities_selected_ids <- unique(entities_selected$entity_id)
      
      # Feature Engineering
      entities_selected <- df_entity %>%
        filter(id %in% entities_selected_ids) %>%
        mutate(country = as.character(lapply(country, country_name))) %>% 
        subset(select = c(id, name, url, description, country, city, founded_in, employees, total_raised))
      
      #
      # Add info from "investment"
      #
      entities_selected <- 
        left_join(entities_selected, df_investment, by = c("id" = "entity_id")) %>% 
        subset(select = c("id", "name", "description", "country", "city", 
                          "founded_in", "employees","total_raised",
                          "amount", "currency", "investment_date")) %>% 
        mutate(investment_date = as.Date(investment_date))
      
      # Filter 
      entities_selected <- entities_selected %>% 
        # 1. by "date" of investment
        filter((investment_date >= input$invest_data_tags_date_fund_dates[1]) & (investment_date <= input$invest_data_tags_date_fund_dates[2])) %>% 
        # 2. change type of feature and modified into 1,000 "K" scale
        # mutate(amount = round(as.numeric(amount)/1000, 3) )
        mutate(amount = as.numeric(amount))
    }
    
    # 5. Get result
    return(entities_selected)
  })


  #
  # 4. NEWS ---------------------------------------------
  #
  
  # Function
  createLink <- function(value) {
    sprintf('<a href="%s" target="_blank" class="btn btn-primary">Info</a>', value)
  }
  
  # 3.1. news_data_all ---------------------------------------------
  
  # 1. Table
  output$dt_news_data_all <- renderDataTable(
    if(input$dt_news_data_all_filter){
      datatable(df_news_data_all_view(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 2, #c(2,4),
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 40 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 40) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = 'top')
    } else {
      datatable(df_news_data_all_view(), options = list(
        searchHighlight = TRUE, 
        pageLength = 25,
        columnDefs = list(list(
          targets = 2, #c(2,4),
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 40 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 40) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = 'none')
    }
    
    
    # datatable(df_news_data_all_view(), options = list(pageLength = 10, searching = TRUE), rownames= FALSE)
  )
  
  # 2. Data frame
  df_news_data_all <- reactive({
    
    # 1. Feature engineering
    df_result <- df_news %>% 
      left_join(df_news_entity, by = c("id" = "news_id")) %>% 
      left_join(df_entity, by = c("entity_id" = "id")) %>% 
      left_join(df_media, by = c("media_id" = "id")) %>% 
      left_join(df_type, by = c("type_id" = "id")) %>% 
      left_join(df_region, by = c("region_id" = "id")) %>%
      subset(select = c(id, title, publish_date, url.x,
                        name.x, name.y, name.x.x, name.y.y)) %>% 
      rename(url = url.x, entity_name = name.x, media_name = name.y, 
             type_name = name.x.x, region_name = name.y.y) %>% 
      mutate(
        # url = paste0("<a href='", url, "' target='_blank'>", url, "</a>"),
        url = createLink(url),
        publish_date = as.Date(publish_date))
    
    # 3. Get result
    return(df_result)
    
  })
  
  # View for DT table
  df_news_data_all_view <- reactive({
    df_news_data_all() %>% 
      subset(select = c(url, title, publish_date, entity_name, # id, 
                        media_name, type_name, region_name))
  })
  
  # 3.2. news_data_tags ---------------------------------------------
  
  # 1. Data frame
  df_news_data_tags <- reactive({
    
    # 1. get list of selected tags from input
    list_of_selected_tags <- input$news_data_tags_tags
    
    # 2. pipeline
    tags_selected <-  df_tag %>% 
      filter(name %in% list_of_selected_tags)
    
    # 3. get news with selected tags
    news_tags_selected <- df_tag_news %>% 
      filter(tag_id %in% tags_selected$id)
    
    # write_feather(news_tags_selected, "entities_tags_selected")
    
    # 4. list of unique tags
    unique_tags <- unique(news_tags_selected$tag_id)
    
    #
    # Check type of filter
    #
    # 4.1. Select entities <OR>
    if (input$news_data_tags_tags_filter == "1"){
      
      # 4.1.1. Feature Engineering
      news_selected <- df_news %>%
        # 1. select news by tags
        filter(id %in% news_tags_selected$news_id) %>% 
        # 2. join
        left_join(df_news_entity, by = c("id" = "news_id")) %>% 
        left_join(df_entity, by = c("entity_id" = "id")) %>% 
        left_join(df_media, by = c("media_id" = "id")) %>% 
        left_join(df_type, by = c("type_id" = "id")) %>% 
        left_join(df_region, by = c("region_id" = "id")) %>%
        # 3. subset
        subset(select = c(url.x, title, publish_date, name.x, 
                          name.y, name.x.x, name.y.y)) %>%
        # 4. rename
        rename(url = url.x, entity_name = name.x, media_name = name.y, 
               type_name = name.x.x, region_name = name.y.y) %>%
        # 5. mutate
        mutate(url = createLink(url),
               publish_date = as.Date(publish_date))
      
    }
    
    # 4.2. Select entities <AND>
    if (input$news_data_tags_tags_filter == "2"){
      
      # 4.2.1. Select news by tags
      news_selected <- news_tags_selected %>%
        group_by(news_id) %>%
        summarise(n = n()) %>%
        filter(n == length(unique_tags))
      
      # 4.2.2. Selected entities from news
      news_selected_ids <- unique(news_selected$news_id)
      
      # 4.2.3. Feature Engineering
      news_selected <- df_news %>%
        # 1. filter
        filter(id %in% news_selected_ids) %>% 
        # 2. join
        left_join(df_news_entity, by = c("id" = "news_id")) %>% 
        left_join(df_entity, by = c("entity_id" = "id")) %>% 
        left_join(df_media, by = c("media_id" = "id")) %>% 
        left_join(df_type, by = c("type_id" = "id")) %>% 
        left_join(df_region, by = c("region_id" = "id")) %>%
        # 3. subset
        subset(select = c(url.x, title, publish_date, name.x, 
                          name.y, name.x.x, name.y.y)) %>%
        # 4. rename
        rename(url = url.x, entity_name = name.x, media_name = name.y, 
               type_name = name.x.x, region_name = name.y.y) %>%
        # 5. mutate
        mutate(url = createLink(url),
               publish_date = as.Date(publish_date))
      
    }
    
    # 5. Get result
    return(news_selected)
    
  })
  
  # 1. Table
  output$dt_news_data_tags <- renderDataTable(
    if(input$dt_news_data_tags_filter){
      datatable(df_news_data_tags(), options = list(
        searchHighlight = TRUE,
        pageLength = 25,
        columnDefs = list(list(
          targets = 2,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 40 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 40) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = 'top')
    } else {
      datatable(df_news_data_tags(), options = list(
        searchHighlight = TRUE,
        pageLength = 25,
        columnDefs = list(list(
          targets = 2,
          render = JS(
            "function(data, type, row, meta) {",
            "return type === 'display' && data.length > 40 ?",
            "'<span title=\"' + data + '\">' + data.substr(0, 40) + '...</span>' : data;",
            "}")
        ))), callback = JS('table.page(3).draw(false);'),
        escape = FALSE,
        filter = 'none')
    }
    
  )
  
  output$btn_download_news_data_tags <- downloadHandler(
    filename = "news_by_tags.csv",
    content = function(file) {
      write.csv(df_news_data_tags(), file)
    }
  )
  
  
}

shinyApp(ui, server)