Home work 4

1. Goal

Repeat Part 1 using FinMind (https://finmindtrade.com) as the data source. The final dataset keeps only ordinary common stocks listed on the Taiwan Stock Exchange (TWSE). ETFs, funds, preferred stocks and other non-common securities are removed.

2. Load packages

# Install once (only if missing):
# install.packages(c("rmarkdown", "knitr", "httr2", "dplyr"))
library(httr2)   # send the web request to the FinMind API
library(dplyr)   # filter / clean / summarise data frames
library(knitr)   # kable() prints nice tables in the HTML

3. Download the data from the FinMind API

FinMind’s dataset for Taiwan stock information is called TaiwanStockInfo. Endpoint: https://api.finmindtrade.com/api/v4/data, parameter dataset=TaiwanStockInfo. Its columns are industry_category, stock_id, stock_name, type and date.

# A token is OPTIONAL. If you have one, set it in the R console (do NOT type it
# into this file):  Sys.setenv(FINMIND_TOKEN = "your_token_here")
token <- Sys.getenv("FINMIND_TOKEN")

# Build the request
req <- request("https://api.finmindtrade.com/api/v4/data") |>
  req_url_query(dataset = "TaiwanStockInfo") |>
  req_timeout(60)

# Add the token only if the user has set one
if (nzchar(token)) req <- req_auth_bearer_token(req, token)

# Send the request. req_perform() stops with an error for HTTP errors (4xx/5xx),
# so we wrap it in tryCatch() to give a readable message.
resp <- tryCatch(
  req_perform(req),
  error = function(e) stop("FinMind request failed: ", conditionMessage(e))
)

# Convert the JSON body into R objects (data part becomes a data frame)
body <- resp_body_json(resp, simplifyVector = TRUE)

# Check the API-level status (FinMind returns status = 200 when OK)
if (is.null(body$status) || body$status != 200) {
  stop("FinMind API error: ", body$msg)
}
if (is.null(body$data) || NROW(body$data) == 0) {
  stop("FinMind returned an empty dataset.")
}

raw <- as_tibble(body$data)

# Check that the columns we need exist
needed <- c("stock_id", "stock_name", "industry_category", "type")
missing_cols <- setdiff(needed, names(raw))
if (length(missing_cols) > 0) {
  stop("Missing expected columns: ", paste(missing_cols, collapse = ", "))
}

n_downloaded <- nrow(raw)
cat("Number of securities downloaded before filtering:", n_downloaded, "\n")

## Number of securities downloaded before filtering: 4340

4. Clean and filter (keep only ordinary common stocks)

# Step 0: make sure the key columns are text and remove extra spaces.
# stock_id MUST stay a character (text), never a number.
df <- raw |>
  mutate(
    stock_id          = trimws(as.character(stock_id)),
    stock_name        = trimws(as.character(stock_name)),
    industry_category = trimws(as.character(industry_category)),
    type              = tolower(trimws(as.character(type)))
  )

# Categories that FinMind uses for things that are NOT ordinary stocks
non_common_categories <- c(
  "ETF", "ETN", "Index", "\u5927\u76e4",          # ETF, ETN, index, 大盤
  "\u53d7\u76ca\u8b49\u5238",                     # 受益證券 (beneficiary certificates / funds)
  "\u5b58\u8a17\u6191\u8b49",                     # 存託憑證 (depositary receipts)
  "\u7279\u5225\u80a1",                           # 特別股 (preferred shares)
  "\u6b0a\u8b49", "\u8a8d\u8cfc\u6b0a\u8b49",     # 權證, 認購權證 (warrants)
  "\u8a8d\u552e\u6b0a\u8b49", "\u725b\u718a\u8b49",
  "\u7ba1\u7406\u80a1\u7968", "Holding", "Others"
)
# Keywords searched inside the CATEGORY text (a second safety net)
category_keywords <- "ETF|ETN|\u53d7\u76ca|\u5b58\u8a17|\u6b0a\u8b49|\u725b\u718a|\u7279\u5225\u80a1|\u6307\u6578"

# Each step removes rows; we count after every step for the report.
step1 <- df    |> filter(type == "twse")                       # listed on TWSE only
step2 <- step1 |> filter(!is.na(stock_id), stock_id != "",
                         !is.na(stock_name), stock_name != "") # remove missing records
step3 <- step2 |> filter(!is.na(industry_category), industry_category != "",
                         !(industry_category %in% non_common_categories),
                         !grepl(category_keywords, industry_category, ignore.case = TRUE))
# Ordinary TWSE stocks have exactly 4 digits, first digit not 0.
# ETFs (e.g. 0050, 00878), preferred shares (e.g. 2882A) and warrants
# (5-6 characters) do not match this pattern.
step4 <- step3 |> filter(grepl("^[1-9][0-9]{3}$", stock_id))

# Remove duplicated stock codes (keep the most recently updated row)
step5 <- step4 |>
  arrange(desc(date)) |>
  distinct(stock_id, .keep_all = TRUE)

# Summary of what each step removed
filter_log <- tibble(
  step = c("Downloaded", "Only type = twse", "Remove missing records",
           "Remove non-common categories (ETF/fund/DR/preferred/...)",
           "Keep 4-digit numeric codes only", "Remove duplicate stock_id"),
  rows_remaining = c(nrow(df), nrow(step1), nrow(step2),
                     nrow(step3), nrow(step4), nrow(step5))
) |>
  mutate(rows_removed = lag(rows_remaining, default = first(rows_remaining)) - rows_remaining)

kable(filter_log, caption = "Rows remaining / removed at each filtering step")

Rows remaining / removed at each filtering step

step	rows_remaining	rows_removed
Downloaded	4340	0
Only type = twse	2404	1936
Remove missing records	2404	0
Remove non-common categories (ETF/fund/DR/preferred/…)	2029	375
Keep 4-digit numeric codes only	1989	40
Remove duplicate stock_id	1216	773

5. Build the final dataset

FinMind’s TaiwanStockInfo does not contain the original listing date. Its date column is only the date FinMind updated the record, so it must not be treated as a listing date. The code uses a listing-date column only if FinMind provides one; otherwise listing_date is left as NA.

# Look for a real listing-date column in the FinMind data (none expected)
date_col <- intersect(c("listing_date", "list_date"), names(step5))

final <- step5 |>
  transmute(
    stock_id,
    name = stock_name,
    listing_date = if (length(date_col) > 0) as.Date(.data[[date_col[1]]]) else as.Date(NA),
    industry = industry_category,
    finmind_update_date = as.Date(date)   # for reference only, NOT the listing date
  ) |>
  arrange(stock_id)

n_final <- nrow(final)
cat("Securities downloaded before filtering:", n_downloaded, "\n")

## Securities downloaded before filtering: 4340

cat("Common stocks remaining after filtering:", n_final, "\n")

## Common stocks remaining after filtering: 1216

cat("Records removed:", n_downloaded - n_final, "\n")

## Records removed: 3124

cat("Rows with missing listing_date:", sum(is.na(final$listing_date)), "\n")

## Rows with missing listing_date: 1216

First 20 rows

kable(head(final, 20))

stock_id	name	listing_date	industry	finmind_update_date
1101	台泥	NA	水泥工業	2026-10-16
1102	亞泥	NA	水泥工業	2026-10-16
1103	嘉泥	NA	水泥工業	2026-10-16
1104	環泥	NA	水泥工業	2026-10-16
1107	建台	NA	其他	2024-12-04
1108	幸福	NA	水泥工業	2026-10-16
1109	信大	NA	水泥工業	2026-10-16
1110	東泥	NA	水泥工業	2026-10-16
1201	味全	NA	食品工業	2026-10-16
1203	味王	NA	食品工業	2026-10-16
1210	大成	NA	食品工業	2026-10-16
1213	大飲	NA	食品工業	2026-10-16
1215	卜蜂	NA	食品工業	2026-10-16
1216	統一	NA	食品工業	2026-10-16
1217	愛之味	NA	食品工業	2026-10-16
1218	泰山	NA	食品工業	2026-10-16
1219	福壽	NA	食品工業	2026-10-16
1220	台榮	NA	食品工業	2026-10-16
1225	福懋油	NA	食品工業	2026-10-16
1227	佳格	NA	食品工業	2026-10-16

Sanity check

If the filters worked, this table should be empty (no ETF/fund/preferred/DR wording left in the names, and no stock code that is not 4 digits).

suspicious <- final |>
  filter(grepl("ETF|ETN|\u57fa\u91d1|\u7279\u5225\u80a1|-DR", name, ignore.case = TRUE) |
         !grepl("^[1-9][0-9]{3}$", stock_id))
kable(suspicious, caption = "Possible leftovers (should be empty)")

Possible leftovers (should be empty)

stock_id	name	listing_date	industry	finmind_update_date
2883	凱基金	NA	金融保險	2026-10-16

6. Number of stocks by industry

industry_summary <- final |>
  count(industry, name = "n_stocks", sort = TRUE)
kable(industry_summary)

industry	n_stocks
電子工業	258
半導體業	75
光電業	62
其他	62
化學生技醫療	57
建材營造	57
電機機械	51
金融保險	46
紡織纖維	43
汽車工業	42
其他電子業	39
通信網路業	35
鋼鐵工業	31
航運業	29
創新板股票	26
化學工業	25
食品工業	24
塑膠工業	23
綠能環保	23
電腦及週邊設備業	22
貿易百貨	20
電器電纜	20
觀光餐旅	19
電子零組件業	19
運動休閒	18
居家生活	11
數位雲端	11
橡膠工業	11
生技醫療業	11
油電燃氣業	9
電子通路業	9
資訊服務業	8
造紙工業	8
水泥工業	7
玻璃陶瓷	5

7. Save the cleaned dataset

# stock_id is text, so leading zeros (if any) are preserved.
write.csv(final, "finmind_tw_listed_stocks.csv",
          row.names = FALSE, na = "", fileEncoding = "UTF-8")
cat("Saved to:", normalizePath("finmind_tw_listed_stocks.csv"), "\n")

## Saved to: C:\Users\Dell\Downloads\finmind_tw_listed_stocks.csv

8. Limitations

• FinMind TaiwanStockInfo has no original listing date. The date column is an update date, so listing_date is NA unless FinMind adds such a column.
• “Listed” means type == "twse" (TWSE main board). OTC (tpex) stocks are excluded.
• Filtering relies on FinMind’s industry_category plus the 4-digit code format. FinMind may rename categories in the future, so check the sanity table above.