If any package is missing, run this once in the R console (it is not evaluated when knitting):
install.packages(c("httr", "jsonlite", "dplyr", "readr"))The TaiwanStockInfo dataset is the primary source. An
API token is optional. To use one, set the environment
variable FINMIND_TOKEN (for example in
~/.Renviron: FINMIND_TOKEN=your_token_here)
and restart R. Never hard-code a token in this file.
api_url <- "https://api.finmindtrade.com/api/v4/data"
token <- Sys.getenv("FINMIND_TOKEN", unset = "")
# Token (if any) is sent as a Bearer header so it never appears in the URL.
req_headers <- if (nzchar(token)) {
add_headers(Authorization = paste("Bearer", token))
} else {
add_headers()
}
resp <- tryCatch(
GET(api_url,
query = list(dataset = "TaiwanStockInfo"),
req_headers,
timeout(60)),
error = function(e) {
stop("FinMind API request failed (network/connection error): ",
conditionMessage(e), call. = FALSE)
}
)
# HTTP-level validation
if (status_code(resp) != 200) {
stop("FinMind API returned HTTP status ", status_code(resp),
". Body: ", substr(content(resp, as = "text", encoding = "UTF-8"), 1, 300),
call. = FALSE)
}
# Parse JSON
body_text <- content(resp, as = "text", encoding = "UTF-8")
parsed <- tryCatch(
fromJSON(body_text, simplifyDataFrame = TRUE),
error = function(e) {
stop("Could not parse the FinMind response as JSON: ",
conditionMessage(e), call. = FALSE)
}
)
# Response-level validation (FinMind reports its own status inside the JSON)
if (is.null(parsed$status) || parsed$status != 200) {
stop("FinMind response status was not 200. Status: ",
ifelse(is.null(parsed$status), "missing", parsed$status),
"; message: ", ifelse(is.null(parsed$msg), "none", parsed$msg),
call. = FALSE)
}
raw_info <- parsed$data
if (is.null(raw_info) || !is.data.frame(raw_info) || nrow(raw_info) == 0) {
stop("FinMind returned an empty 'data' payload for TaiwanStockInfo.", call. = FALSE)
}
expected_cols <- c("industry_category", "stock_id", "stock_name", "type", "date")
missing_cols <- setdiff(expected_cols, names(raw_info))
if (length(missing_cols) > 0) {
stop("Expected column(s) missing from TaiwanStockInfo: ",
paste(missing_cols, collapse = ", "),
". Columns received: ", paste(names(raw_info), collapse = ", "),
call. = FALSE)
}
raw_info <- tibble::as_tibble(raw_info)
cat("Rows downloaded:", nrow(raw_info), "\n")## Rows downloaded: 4340cat("Columns received:", paste(names(raw_info), collapse = ", "), "\n")## Columns received: industry_category, stock_id, stock_name, type, dateThe date column in TaiwanStockInfo is a
dataset/update date, not a listing date, so it is not
used for listing_date. The chunk below inspects the columns
actually returned by the API for anything that looks like a listing/IPO
field.
listing_like_cols <- grep("list|ipo|issue", names(raw_info),
ignore.case = TRUE, value = TRUE)
if (length(listing_like_cols) == 0) {
cat("No listing-date-like column found in TaiwanStockInfo.\n")
} else {
cat("Possible listing-related column(s) found:",
paste(listing_like_cols, collapse = ", "), "\n")
}## No listing-date-like column found in TaiwanStockInfo.# Only an explicitly named `listing_date` column returned by the API itself
# would be used. Otherwise listing_date is set to NA (never guessed).
has_listing_date <- "listing_date" %in% names(raw_info)
cat("Explicit 'listing_date' column present:", has_listing_date, "\n")## Explicit 'listing_date' column present: FALSEOther FinMind datasets (for example TaiwanStockPrice)
only tell us the first trading day within FinMind’s own coverage
window, which is not the true listing date for older companies.
They are deliberately not used as a substitute.
Filtering rules:
type == "twse" (case-insensitive, whitespace
trimmed).NA, and drop
rows with missing stock_id or stock_name.stock_id values, keeping the row with
the most recent date (ties broken alphabetically by
industry_category so the result is deterministic).stock_id must be exactly four digits starting with 1-9
(^[1-9][0-9]{3}$). This removes ETFs
(e.g. 0050, 00878), warrants and convertible
bonds (5-6 characters), and preferred shares (a letter suffix such as
2882A).industry_category must not match ETF / fund / index /
depositary-receipt / bond / warrant keywords.stock_name must not contain ETF,
ETN, or end in -DR (case-insensitive). Chinese
characters that might appear in ordinary company names (such as 特) are
intentionally not used as a filter.industry_exclude_regex <- paste(
c("ETF", "ETN", "Index", "Fund", "Warrant", "Bond", "Depositary",
"大盤", "受益證券", "存託憑證", "基金", "債券", "權證", "所有證券"),
collapse = "|"
)
name_exclude_regex <- "ETF|ETN|-DR$"
common_id_regex <- "^[1-9][0-9]{3}$"
twse_all <- raw_info %>%
mutate(
across(c(stock_id, stock_name, industry_category, type),
~ trimws(as.character(.x))),
across(c(stock_id, stock_name, industry_category, type),
~ na_if(.x, "")),
date_parsed = suppressWarnings(as.Date(date))
) %>%
filter(tolower(type) == "twse")
if (nrow(twse_all) == 0) {
stop("No rows with type == 'twse' were found. Values of 'type' received: ",
paste(unique(raw_info$type), collapse = ", "), call. = FALSE)
}
twse_dedup <- twse_all %>%
filter(!is.na(stock_id), !is.na(stock_name)) %>%
arrange(stock_id, desc(date_parsed), industry_category) %>%
distinct(stock_id, .keep_all = TRUE)
classified <- twse_dedup %>%
mutate(
exclusion_reason = case_when(
!grepl(common_id_regex, stock_id) ~ "stock_id not a 4-digit common-stock code",
grepl(industry_exclude_regex, industry_category, ignore.case = TRUE) ~ "excluded industry category",
grepl(name_exclude_regex, stock_name, ignore.case = TRUE) ~ "excluded name pattern",
TRUE ~ "kept"
)
)
final_data <- classified %>%
filter(exclusion_reason == "kept") %>%
transmute(
stock_id = as.character(stock_id),
name = stock_name,
listing_date = if (has_listing_date) {
suppressWarnings(as.Date(.data$listing_date))
} else {
as.Date(NA)
},
industry = industry_category # NA stays NA; nothing is invented
) %>%
arrange(stock_id)# Rows removed at each stage
stage_summary <- tibble::tibble(
stage = c("Rows downloaded",
"type == 'twse' rows",
"After removing missing id/name and duplicate stock_id",
"Retained common stocks"),
rows = c(nrow(raw_info), nrow(twse_all), nrow(twse_dedup), nrow(final_data))
)
print(stage_summary)## # A tibble: 4 × 2
## stage rows
## <chr> <int>
## 1 Rows downloaded 4340
## 2 type == 'twse' rows 2404
## 3 After removing missing id/name and duplicate stock_id 1628
## 4 Retained common stocks 1216cat("\nExclusion reasons (after de-duplication):\n")##
## Exclusion reasons (after de-duplication):print(count(classified, exclusion_reason, sort = TRUE))## # A tibble: 3 × 2
## exclusion_reason n
## <chr> <int>
## 1 kept 1216
## 2 stock_id not a 4-digit common-stock code 401
## 3 excluded industry category 11cat("\nIndustry categories among EXCLUDED rows (check these look right):\n")##
## Industry categories among EXCLUDED rows (check these look right):print(classified %>%
filter(exclusion_reason != "kept") %>%
count(exclusion_reason, industry_category, sort = TRUE),
n = 40)## # A tibble: 14 × 3
## exclusion_reason industry_category n
## <chr> <chr> <int>
## 1 stock_id not a 4-digit common-stock code ETF 272
## 2 stock_id not a 4-digit common-stock code Index 30
## 3 stock_id not a 4-digit common-stock code ETN 28
## 4 stock_id not a 4-digit common-stock code 存託憑證 25
## 5 stock_id not a 4-digit common-stock code 金融保險 24
## 6 excluded industry category 存託憑證 11
## 7 stock_id not a 4-digit common-stock code 受益證券 8
## 8 stock_id not a 4-digit common-stock code 其他 6
## 9 stock_id not a 4-digit common-stock code 電子工業 3
## 10 stock_id not a 4-digit common-stock code 塑膠工業 1
## 11 stock_id not a 4-digit common-stock code 大盤 1
## 12 stock_id not a 4-digit common-stock code 水泥工業 1
## 13 stock_id not a 4-digit common-stock code 汽車工業 1
## 14 stock_id not a 4-digit common-stock code 鋼鐵工業 1cat("\nSample of excluded rows whose stock_id looks like 4 digits (possible false exclusions):\n")##
## Sample of excluded rows whose stock_id looks like 4 digits (possible false exclusions):print(classified %>%
filter(exclusion_reason != "kept", grepl("^[0-9]{4}$", stock_id)) %>%
select(stock_id, stock_name, industry_category, exclusion_reason),
n = 30)## # A tibble: 25 × 4
## stock_id stock_name industry_category exclusion_reason
## <chr> <chr> <chr> <chr>
## 1 0050 元大台灣50 ETF stock_id not a 4-digit common-stock …
## 2 0051 元大中型100 ETF stock_id not a 4-digit common-stock …
## 3 0052 富邦科技 ETF stock_id not a 4-digit common-stock …
## 4 0053 元大科技市值 ETF stock_id not a 4-digit common-stock …
## 5 0054 元大台商50 ETF stock_id not a 4-digit common-stock …
## 6 0055 元大MSCI金融 ETF stock_id not a 4-digit common-stock …
## 7 0056 元大高股息 ETF stock_id not a 4-digit common-stock …
## 8 0057 富邦摩台 ETF stock_id not a 4-digit common-stock …
## 9 0058 富邦發達 ETF stock_id not a 4-digit common-stock …
## 10 0059 富邦金融 ETF stock_id not a 4-digit common-stock …
## 11 0060 新台灣 ETF stock_id not a 4-digit common-stock …
## 12 0061 元大寶滬深 ETF stock_id not a 4-digit common-stock …
## 13 0080 恒中國 ETF stock_id not a 4-digit common-stock …
## 14 0081 恒香港 ETF stock_id not a 4-digit common-stock …
## 15 9101 福雷電 存託憑證 excluded industry category
## 16 9102 東亞科 存託憑證 excluded industry category
## 17 9103 美德醫療-DR 存託憑證 excluded industry category
## 18 9104 萬宇科 存託憑證 excluded industry category
## 19 9105 泰金寶-DR 存託憑證 excluded industry category
## 20 9106 新焦點-DR 存託憑證 excluded industry category
## 21 9110 越南控-DR 存託憑證 excluded industry category
## 22 9136 巨騰-DR 存託憑證 excluded industry category
## 23 9151 旺旺 存託憑證 excluded industry category
## 24 9157 陽光能源-DR 存託憑證 excluded industry category
## 25 9188 精熙-DR 存託憑證 excluded industry categorycat("\nAll distinct industry categories among KEPT rows:\n")##
## All distinct industry categories among KEPT rows:print(sort(unique(classified$industry_category[classified$exclusion_reason == "kept"])))## [1] "光電業" "其他" "其他電子業" "創新板股票" "化學工業"
## [6] "化學生技醫療" "半導體業" "塑膠工業" "居家生活" "建材營造"
## [11] "數位雲端" "橡膠工業" "水泥工業" "汽車工業" "油電燃氣業"
## [16] "玻璃陶瓷" "生技醫療業" "紡織纖維" "綠能環保" "航運業"
## [21] "觀光餐旅" "貿易百貨" "資訊服務業" "通信網路業" "造紙工業"
## [26] "運動休閒" "金融保險" "鋼鐵工業" "電器電纜" "電子工業"
## [31] "電子通路業" "電子零組件業" "電機機械" "食品工業"# Stock IDs that appeared under more than one industry in the raw TWSE data
multi_industry_ids <- twse_all %>%
filter(!is.na(stock_id)) %>%
group_by(stock_id) %>%
summarise(n_industries = n_distinct(industry_category, na.rm = TRUE),
.groups = "drop") %>%
filter(n_industries > 1)
cat("\nStock IDs with >1 industry category in the raw data:",
nrow(multi_industry_ids),
"(most recent 'date' row was kept for each)\n")##
## Stock IDs with >1 industry category in the raw data: 715 (most recent 'date' row was kept for each)# Hard checks
stopifnot(
"Final data must have exactly four columns in the required order" =
identical(names(final_data), c("stock_id", "name", "listing_date", "industry")),
"stock_id must be character" = is.character(final_data$stock_id),
"Duplicate stock_id values found" = !anyDuplicated(final_data$stock_id),
"Missing/empty stock_id found" = !any(is.na(final_data$stock_id) | final_data$stock_id == ""),
"Missing/empty name found" = !any(is.na(final_data$name) | final_data$name == ""),
"Non common-stock IDs remain" = all(grepl(common_id_regex, final_data$stock_id)),
"Excluded industry categories remain" =
!any(grepl(industry_exclude_regex, final_data$industry, ignore.case = TRUE)),
"Excluded name patterns remain" =
!any(grepl(name_exclude_regex, final_data$name, ignore.case = TRUE))
)
# Market-type check on the source rows that produced the final data
market_check <- twse_dedup %>%
filter(stock_id %in% final_data$stock_id) %>%
count(type)
print(market_check)## # A tibble: 1 × 2
## type n
## <chr> <int>
## 1 twse 1216stopifnot(all(tolower(market_check$type) == "twse"))
cat("\nMissing values per column:\n")##
## Missing values per column:print(colSums(is.na(final_data)))## stock_id name listing_date industry
## 0 0 1216 0cat("\nAll validation checks passed.\n")##
## All validation checks passed.n_retained <- nrow(final_data)
n_missing_listing <- sum(is.na(final_data$listing_date))
n_missing_industry <- sum(is.na(final_data$industry))
cat("Total retained common stocks:", n_retained, "\n")## Total retained common stocks: 1216cat("Missing listing dates:", n_missing_listing,
sprintf("(%.1f%%)\n", 100 * n_missing_listing / n_retained))## Missing listing dates: 1216 (100.0%)cat("Missing industries:", n_missing_industry, "\n")## Missing industries: 0First 10 rows:
head(final_data, 10)output_file <- "finmind_twse_common_stocks.csv"
tryCatch(
readr::write_csv(final_data, output_file, na = "NA"), # readr always writes UTF-8
error = function(e) {
stop("Could not write ", output_file, ": ", conditionMessage(e), call. = FALSE)
}
)
output_path <- normalizePath(output_file, mustWork = FALSE)cat(sprintf("**Output file:** `%s` \n", output_path))Output file:
C:\Users\deegi\Downloads\finmind_twse_common_stocks.csv
cat(sprintf("**Rows written:** %d\n\n", nrow(final_data)))Rows written: 1216
cat("**Limitations:**\n\n")Limitations:
cat("- `listing_date` is `NA` unless the API returns an explicit `listing_date` column. ",
"The `date` field in `TaiwanStockInfo` is a dataset date and was not used.\n", sep = "")listing_date is NA unless the API returns
an explicit listing_date column. The date
field in TaiwanStockInfo is a dataset date and was not
used.cat("- `TaiwanStockInfo` has no security-type field, so common stocks are identified by a ",
"stock-code convention (four digits, 1-9 first) plus industry and name keyword exclusions. ",
"This is a defensible heuristic, not a guaranteed classification.\n", sep = "")TaiwanStockInfo has no security-type field, so common
stocks are identified by a stock-code convention (four digits, 1-9
first) plus industry and name keyword exclusions. This is a defensible
heuristic, not a guaranteed classification.cat("- Some stock IDs may appear under several industry categories in the source; ",
"the most recent row was kept.\n", sep = "")cat("- Results reflect FinMind's data at the time of the run and may change.\n")Common-stock filter. FinMind’s
TaiwanStockInfo has no field that states whether a security
is a common share, preferred share, ETF, etc. The filter therefore uses
(1) the Taiwan stock-code convention, where ordinary listed companies
have four-digit codes and other instruments use longer codes or letter
suffixes, (2) keyword exclusion on industry_category, and
(3) a narrow name check (ETF, ETN,
-DR). Ambiguous Chinese characters are not used for
filtering, so ordinary companies are not removed by accident. Edge cases
(unusual codes, newly created instrument classes) could still be
misclassified.
Listing dates. No reliable listing-date field was
assumed to exist in FinMind. The script checks the returned columns at
run time and sets listing_date to NA
otherwise. First-trade dates derived from price data were intentionally
not substituted because they reflect FinMind’s coverage start, not the
real listing date.
Industry. Taken directly from
industry_category; missing or empty values stay
NA.
Reproducibility. This document does not store any API result. Row counts and the CSV are produced only when you knit it.