test

# install.packages(c("rvest", "httr", "jsonlite", "dplyr", "stringr", "tibble"))
library(rvest)
library(httr)
library(jsonlite)
library(dplyr)

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

library(stringr)
library(tibble)

PART 1: TWSE ISIN (strMode = 2)

cat("-> Fetching data from TWSE ISIN lookup page...\n")

## -> Fetching data from TWSE ISIN lookup page...

url_twse <- "https://isin.twse.com.tw/isin/C_public.jsp?strMode=2"
response_twse <- GET(url_twse, user_agent("Mozilla/5.0"))
stop_for_status(response_twse)

# page is Big5/CP950
page <- read_html(content(response_twse, as = "raw"), encoding = "CP950")

# Select the data table explicitly instead of taking the first table
tbls <- page %>% html_elements("table.h4")
if (length(tbls) == 0) tbls <- page %>% html_elements("table")
cat("   Tables found:", length(tbls), "\n")

##    Tables found: 1

stopifnot(length(tbls) > 0)

# header = FALSE so the first row stays as data (we promote it ourselves)
df_raw <- tbls[[1]] %>% html_table(header = FALSE, fill = TRUE)
stopifnot(ncol(df_raw) > 0, nrow(df_raw) > 1)

# First row is the header row
names(df_raw) <- as.character(df_raw[1, ])
df_raw <- df_raw[-1, ]

# code and name are separated by a FULL-WIDTH space (U+3000)
code_name <- as.character(df_raw[[1]])
stock_id  <- str_trim(str_extract(code_name, "^[^\u3000 ]+"))
name      <- str_trim(str_replace(code_name, "^[^\u3000 ]+[\u3000 ]+", ""))

df_twse_clean <- tibble(
  stock_id     = stock_id,
  name         = name,
  listing_date = as.character(df_raw[[3]]),
  industry     = as.character(df_raw[[5]])
) %>%
  filter(!is.na(stock_id), str_detect(stock_id, "^\\d{4}$")) %>%
  filter(!str_detect(name, "特別股")) %>%
  filter(!is.na(industry), industry != "")   # ETFs have an empty industry

cat(sprintf("   Extracted %d common stocks from TWSE.\n\n", nrow(df_twse_clean)))

##    Extracted 1085 common stocks from TWSE.

PART 2: FinMind API

cat("-> Fetching data from FinMind API...\n")

## -> Fetching data from FinMind API...

response_fm <- GET("https://api.finmindtrade.com/api/v4/data",
                   query = list(dataset = "TaiwanStockInfo"))
parsed <- fromJSON(content(response_fm, as = "text", encoding = "UTF-8"))

if (is.null(parsed$msg) || parsed$msg != "success") {
  stop("Failed to fetch data from FinMind API: ", parsed$msg)
}

df_fm_clean <- parsed$data %>%
  rename(name = stock_name, industry = industry_category, listing_date = date) %>%
  filter(type == "twse") %>%
  filter(str_detect(stock_id, "^\\d{4}$")) %>%
  filter(!str_detect(industry, "ETF|ETN|Index|Beneficiary|受益|基金|大盤")) %>%
  filter(!str_detect(name, "特別股")) %>%
  distinct(stock_id, .keep_all = TRUE) %>%
  select(stock_id, name, listing_date, industry)

# Sanity check: compare TWSE and FinMind listing dates
chk <- inner_join(df_twse_clean, df_fm_clean, by = "stock_id", suffix = c("_twse", "_fm")) %>%
  mutate(same = as.Date(gsub("/", "-", listing_date_twse), optional = TRUE) ==
                as.Date(listing_date_fm, optional = TRUE))
cat(sprintf("   Listing date matches TWSE for %.1f%% of shared stocks.\n",
            100 * mean(chk$same, na.rm = TRUE)))

##    Listing date matches TWSE for 0.0% of shared stocks.

cat(sprintf("   Extracted %d common stocks from FinMind.\n\n", nrow(df_fm_clean)))

##    Extracted 1226 common stocks from FinMind.

PART 3: Export

write.csv(df_twse_clean, "twse_common_stocks.csv", row.names = FALSE, fileEncoding = "UTF-8")
write.csv(df_fm_clean,   "finmind_common_stocks.csv", row.names = FALSE, fileEncoding = "UTF-8")
cat("Done.\n")

## Done.