hw4

# Install and load necessary packages
if (!require("pacman")) install.packages("pacman")

## Loading required package: pacman

pacman::p_load(tidyverse, rvest, jsonlite)

# --- TASK 1: Download from TWSE's ISIN lookup page ---
twse_url <- "https://isin.twse.com.tw/isin/C_public.jsp?strMode=2"

# FIX: Specify encoding and the exact data table class "h4"
twse_html <- read_html(twse_url, encoding = "MS950")
twse_table <- twse_html %>% html_node("table.h4") %>% html_table(fill = TRUE)

# Clean the dataset
colnames(twse_table) <- as.character(twse_table[1, ])
twse_data <- twse_table[-1, ]

# Separate ID and Name, extract listing date and industry
twse_clean <- twse_data %>%
  select(1, 3, 6) %>% 
  rename(Stock_ID_Name = 1, Listing_Date = 2, Industry = 3) %>%
  filter(Industry != "") %>%
  # The space in the TWSE table is a full-width space " "
  separate(Stock_ID_Name, into = c("stock_id", "name"), sep = " ", extra = "merge") %>%
  # Filter out ETFs, funds, and preferred stocks by keeping only 4-digit stock IDs
  filter(nchar(stock_id) == 4)

## Warning: Expected 2 pieces. Missing pieces filled with `NA` in 38318 rows [1, 2, 3, 4,
## 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].

print("Task 1 Results (TWSE):")

## [1] "Task 1 Results (TWSE):"

head(twse_clean)

## # A tibble: 1 × 4
##   stock_id name  Listing_Date Industry
##   <chr>    <chr> <chr>        <chr>   
## 1 ªÑ²¼     <NA>  ªÑ²¼         ªÑ²¼

# --- TASK 2: Download data provided by FinMind ---
finmind_url <- "https://api.finmindtrade.com/api/v4/data?dataset=TaiwanStockInfo"

# Fetch JSON data from FinMind API
finmind_json <- fromJSON(finmind_url)
finmind_data <- finmind_json$data

# Clean and filter the dataset
finmind_clean <- finmind_data %>%
  select(stock_id, name = stock_name, Listing_Date = date, Industry = industry_category) %>%
  # Exclude ETFs, funds, and preferred stocks by keeping only 4-digit stock IDs
  filter(nchar(stock_id) == 4)

print("Task 2 Results (FinMind):")

## [1] "Task 2 Results (FinMind):"

head(finmind_clean)

##   stock_id      name Listing_Date         Industry
## 1     5481      新華   2020-06-03     電子零組件業
## 2     5450    寶聯通   2020-06-03 電腦及週邊設備業
## 3     3629  地心引力   2020-06-03           光電業
## 4     3687    歐買尬   2020-06-03       文化創意業
## 5     6238      勝麗   2020-06-13       其他電子類
## 6     6497 亞獅康-KY   2020-08-27       生技醫療業