Lokasi File: d:/projek/riskysi/uts_report_publish.Rmd

Berikut adalah implementasi teknis pemrosesan dan scraping data di R yang telah disusun secara sistematis.

SECTION 1: CASE STUDY E-COMMERCE (R IMPLEMENTATION)

1. Data Collection

files <- c('ecommerce.csv', 'ecommerce.json', 'ecommerce.txt', 'ecommerce.xlsx', 'ecommerce.xml')
data_list <- list()
base_cols <- NULL

for (f in files) {
  message("Reading file: ", f)
  
  if (grepl(".csv$", f)) {
    df <- read.csv(f)
  } else if (grepl(".json$", f)) {
    df <- fromJSON(f)
  } else if (grepl(".txt$", f)) {
    df <- read.csv(f, sep = "|") # Corrected delimiter to pipe
  } else if (grepl(".xlsx$", f)) {
    df <- read_excel(f)
  } else if (grepl(".xml$", f)) {
    xml_data <- read_xml(f)
    records <- xml_find_all(xml_data, ".//Record") # Correct tag casing
    df <- map_df(records, function(x) {
      children <- xml_children(x)
      set_names(xml_text(children), xml_name(children)) %>% as.list()
    })
  }
  
  # Requirement: Tampilkan info dasar
  print(paste("File:", f, "| Rows:", nrow(df), "| Cols:", ncol(df)))
  
  # Requirement: IF/IF-ELSE untuk cek struktur
  current_cols <- sort(colnames(df))
  if (is.null(base_cols)) {
    base_cols <- current_cols
    print("-> Ready to merge (Base structure set)")
  } else {
    if (all(current_cols == base_cols)) {
      print("-> Ready to merge")
    } else {
      print("-> Need adjustment")
    }
  }
  
  data_list[[f]] <- df
}

## Reading file: ecommerce.csv

## [1] "File: ecommerce.csv | Rows: 2000 | Cols: 22"
## [1] "-> Ready to merge (Base structure set)"

## Reading file: ecommerce.json

## [1] "File: ecommerce.json | Rows: 2000 | Cols: 22"
## [1] "-> Ready to merge"

## Reading file: ecommerce.txt

## [1] "File: ecommerce.txt | Rows: 2000 | Cols: 22"
## [1] "-> Ready to merge"

## Reading file: ecommerce.xlsx

## [1] "File: ecommerce.xlsx | Rows: 2000 | Cols: 22"
## [1] "-> Ready to merge"

## Reading file: ecommerce.xml

## [1] "File: ecommerce.xml | Rows: 2000 | Cols: 22"
## [1] "-> Ready to merge"

### Gabungkan data
main_df_r <- do.call(rbind, data_list)
print(paste("Total Merged Data:", nrow(main_df_r), "rows"))

## [1] "Total Merged Data: 10000 rows"

2. Data Cleaning (Explicit Logic)

### Requirement: WAJIB IF & LOOP
### Kita akan melakukan iterasi baris demi baris untuk menunjukkan logika
for (i in 1:nrow(main_df_r)) {
  
  # 1. Standardisasi Platform
  plat <- trimws(tolower(as.character(main_df_r$platform[i])))
  if (plat == "shopee") {
    main_df_r$platform[i] <- "Shopee"
  } else if (plat == "tokopedia" || plat == "tokped") {
    main_df_r$platform[i] <- "Tokopedia"
  } else if (plat == "tiktok shop") {
    main_df_r$platform[i] <- "TikTok Shop"
  }
  
  # 2. Cleaning harga/sales
  clean_num <- function(val) {
    val_str <- as.character(val)
    if (grepl("Rp", val_str)) {
      val_str <- gsub("Rp", "", val_str)
      val_str <- gsub("\\.", "", val_str)
      val_str <- gsub(",", ".", val_str)
    }
    num <- as.numeric(trimws(val_str))
    if (is.na(num) || num < 0) return(0)
    return(num)
  }
  
  main_df_r$unit_price[i] <- clean_num(main_df_r$unit_price[i])
  main_df_r$net_sales[i] <- clean_num(main_df_r$net_sales[i])
  
  # 3. Missing Value (Payment Method)
  pm <- trimws(as.character(main_df_r$payment_method[i]))
  if (pm == "" || is.na(pm) || pm == "NA" || pm == "nan") {
    main_df_r$payment_method[i] <- "Unknown"
  }
  
  # Requirement: Imputasi customer_rating (WAJIB IF)
  cr <- main_df_r$customer_rating[i]
  if (is.na(cr) || cr == "" || cr == "NA") {
    # Default ke mode atau nilai logis (misal: 4.0)
    main_df_r$customer_rating[i] <- 4.0
  }
  
  # 4. Standardisasi Order Status
  st <- trimws(tolower(as.character(main_df_r$order_status[i])))
  if (st == "delivered" || st == "completed") {
    main_df_r$order_status[i] <- "Completed"
  } else if (st == "cancelled" || st == "cancel" || st == "batal") {
    main_df_r$order_status[i] <- "Cancelled"
  }
}

### 5. WAJIB LOOPING untuk 3 kolom sekaligus
target_cols <- c("product_name", "category", "region")
for (col in target_cols) {
  main_df_r[[col]] <- trimws(as.character(main_df_r[[col]]))
}

message("Data Cleaning Complete.")

## Data Cleaning Complete.

3. Conditional Logic

main_df_r$is_high_value <- NA
main_df_r$order_priority <- NA
main_df_r$valid_transaction <- NA

for (i in 1:nrow(main_df_r)) {
  sales <- as.numeric(main_df_r$net_sales[i])
  
  # Logic 1: is_high_value
  if (sales > 1000000) {
    main_df_r$is_high_value[i] <- "Yes"
  } else {
    main_df_r$is_high_value[i] <- "No"
  }
  
  # Logic 2: order_priority (WAJIB NESTED IF)
  if (sales > 1000000) {
    main_df_r$order_priority[i] <- "High"
  } else {
    if (sales >= 500000) {
      main_df_r$order_priority[i] <- "Medium"
    } else {
      main_df_r$order_priority[i] <- "Low"
    }
  }
  
  # Logic 3: valid_transaction
  if (main_df_r$order_status[i] == "Cancelled") {
    main_df_r$valid_transaction[i] <- "Invalid"
  } else {
    main_df_r$valid_transaction[i] <- "Valid"
  }
}

head(main_df_r)

##                 order_id order_date  ship_date    platform    category
## ecommerce.csv.1 ORD00612 2024-04-19 2024/04/24   Tokopedia home living
## ecommerce.csv.2 ORD00112 2024/01/24 29/01/2024 TikTok Shop Electronics
## ecommerce.csv.3 ORD01186 2024-06-12 06-19-2024   Tokopedia     Fashion
## ecommerce.csv.4 ORD01511 2024/08/07 08-14-2024   Tokopedia home living
## ecommerce.csv.5 ORD00772 2024-12-08              Tokopedia      Beauty
## ecommerce.csv.6 ORD00880 2024/04/06                 blibli      Beauty
##                   product_name unit_price quantity gross_sales      campaign
## ecommerce.csv.1     Table Lamp     188905        4      755620    Flash Sale
## ecommerce.csv.2     Power Bank    1476873        1     1476873    Normal Day
## ecommerce.csv.3    Women Dress     231072        2      462144 Mega Campaign
## ecommerce.csv.4 Vacuum Cleaner     512063        3     1536189    Flash Sale
## ecommerce.csv.5    Body Lotion     221586        2      443172   Payday Sale
## ecommerce.csv.6       Lip Tint     297973        8     2383784    Normal Day
##                 voucher_code discount_pct discount_value shipping_cost
## ecommerce.csv.1       DISC10           10          75562         12000
## ecommerce.csv.2         NONE            0              0         18000
## ecommerce.csv.3       DISC20           20          92429         15000
## ecommerce.csv.4       DISC10           10         153619             0
## ecommerce.csv.5       DISC15           15      Rp 66.476             0
## ecommerce.csv.6         NONE            0              0         12000
##                 net_sales  payment_method customer_segment     region
## ecommerce.csv.1    680058        E-Wallet              VIP     Bekasi
## ecommerce.csv.2   1476873            cod         Returning   Makassar
## ecommerce.csv.3    369715 Virtual Account        Returning   Surabaya
## ecommerce.csv.4   1382570   Transfer Bank              New Yogyakarta
## ecommerce.csv.5    376696             COD        Returning   Surabaya
## ecommerce.csv.6         0     credit card              VIP   Makassar
##                 stock_status order_status customer_rating priority_flag
## ecommerce.csv.1     Preorder    Completed               5             Y
## ecommerce.csv.2     In Stock    Completed               5             N
## ecommerce.csv.3     In Stock    Completed               3           Yes
## ecommerce.csv.4     In Stock    Completed               4            No
## ecommerce.csv.5     In Stock    Completed               5        normal
## ecommerce.csv.6    Low Stock    Cancelled               4           Yes
##                 is_high_value order_priority valid_transaction
## ecommerce.csv.1            No         Medium             Valid
## ecommerce.csv.2           Yes           High             Valid
## ecommerce.csv.3            No            Low             Valid
## ecommerce.csv.4           Yes           High             Valid
## ecommerce.csv.5            No            Low             Valid
## ecommerce.csv.6            No            Low           Invalid

SECTION 2: WEB SCRAPING - R PROGRAMMING (WAJIB LOOPING & IF-ELSE)

1. Hockey Teams (Pagination & Form Handling)

hockey_results <- data.frame()
search_query <- "B" # Demo: Handling form/query (Searching for teams starting with B)

for (p in 1:3) { 
  # Pagination + Form query parameter
  url <- paste0("https://www.scrapethissite.com/pages/forms/?page_num=", p, "&q=", search_query)
  page <- read_html(url)
  
  names <- page %>% html_nodes(".name") %>% html_text(trim = TRUE)
  years <- page %>% html_nodes(".year") %>% html_text(trim = TRUE)
  pts <- page %>% html_nodes(".pct") %>% html_text(trim = TRUE)
  
  if (length(names) > 0) {
    page_df <- data.frame(team_name = names, year = years, points = pts)
    page_df$data_status <- "Complete"
    hockey_results <- rbind(hockey_results, page_df)
  }
}
write.csv(hockey_results, "hockey_teams.csv", row.names = FALSE)
print(paste("Scraped", nrow(hockey_results), "hockey teams with query:", search_query))

## [1] "Scraped 75 hockey teams with query: B"

2. Turtles All the Way Down (iFrames)

### Access iframe src directly
turtle_url <- "https://www.scrapethissite.com/pages/frames/?frame=i"
page <- read_html(turtle_url)

family <- page %>% html_nodes("h3.family-name") %>% html_text(trim = TRUE)
desc <- page %>% html_nodes(".description") %>% html_text(trim = TRUE)
if (length(desc) == 0) desc <- rep("No description", length(family))
if (length(desc) < length(family)) desc <- c(desc, rep("No description", length(family) - length(desc)))

more_info <- page %>% html_nodes(".lead") %>% html_text(trim = TRUE)
if (length(more_info) == 0) more_info <- rep("No additional info", length(family))
if (length(more_info) < length(family)) more_info <- c(more_info, rep("No additional info", length(family) - length(more_info)))

turtle_results <- data.frame(family_name = family, description = desc, additional_info = more_info)
turtle_results$data_status <- ifelse(turtle_results$family_name == "", "Incomplete", "Complete")

write.csv(turtle_results, "turtles.csv", row.names = FALSE)

SECTION C: DATA CLEANING (WEB SCRAPING) - WAJIB LOOP & IF

print("Starting Section C Data Cleaning for scraped data...")

## [1] "Starting Section C Data Cleaning for scraped data..."

### 1. Clean Hockey Results
for (i in 1:nrow(hockey_results)) {
  # Trim spaces
  hockey_results$team_name[i] <- trimws(as.character(hockey_results$team_name[i]))
  
  # Type conversion with IF
  yr <- as.character(hockey_results$year[i])
  if (!is.na(yr) && yr != "") {
    hockey_results$year[i] <- as.integer(yr)
  }
}
### Validasi dan Remove Duplikat
hockey_results <- unique(hockey_results)

### 2. Clean Turtles Results
for (i in 1:nrow(turtle_results)) {
  # Proper Case for Family Name
  turtle_results$family_name[i] <- tools::toTitleCase(tolower(trimws(as.character(turtle_results$family_name[i]))))
  
  # Bersihkan whitespace berlebih di column description
  desc <- as.character(turtle_results$description[i])
  if (!is.na(desc)) {
    # Remove multiple spaces/newlines
    desc <- gsub("\\s+", " ", desc)
    turtle_results$description[i] <- trimws(desc)
  }
}
turtle_results <- unique(turtle_results)

print("Scraping Data Cleaning Complete. Resaving CSVs...")

## [1] "Scraping Data Cleaning Complete. Resaving CSVs..."

write.csv(hockey_results, "hockey_teams.csv", row.names = FALSE)
write.csv(turtle_results, "turtles.csv", row.names = FALSE)