Load Packages and Functions

library(tidyverse)
library(jtools)

# NOTE:
# THE .Rdata file with the data for this assignment also loads the functions I suggest you use, but I am leaving the lines below (commented out) in case you need to get the functions that way for whatever reason (or in the future)

#source("https://gist.github.com/dankatz00/8ee9d546b2b3f781c0973de5d374ea09/raw/smart_ols.R")
#source("https://gist.github.com/dankatz00/8d3725b536a3a0370858ace921f5615a/raw/smart_logit.R")
#source("https://gist.github.com/dankatz00/05012a2e85b839f2e181d0c67b940df0/raw/smart_validation.R")

Load Data

Load in the “Unit_2_HW_Data.RData” to get the data (and functions) for this analysis.

Use q1_and_q2_data for Questions 1 and 2, use q3_data for Question 3, …

#——————– # PREDICTOR SELECTION #——————–

Question 1:

A company wants to understand how their spending, costs, competitors’ spending, and consumers’ behaviors impact their total revenue.

First, run a regression for total revenue using all other available data. Check for multicollinearity and overfitting. Are there any variables that should be removed from the regression for either/both of these reasons? Why or why not?

Note: while not strictly necessary, I recommend using smart_ols()

Q1 Code

# Load required packages
library(tidyverse)

# The data for Q1 is in q1_and_q2_data
str(q1_and_q2_data)
'data.frame':   100 obs. of  11 variables:
 $ competitor_ad_spend   : num  44395 47698 65587 50705 51293 ...
 $ influencer_spend      : num  34765 34419 50068 40180 40946 ...
 $ ad_spend              : num  21533 21746 29864 21365 25807 ...
 $ web_traffic           : num  539332 531152 825671 619823 698729 ...
 $ discount_percent      : num  14.63 9.16 11.83 14.86 18.35 ...
 $ customer_service_spend: num  4398 4006 6027 5751 3491 ...
 $ avg_star_rating       : num  4.72 4.77 3.98 4.44 3.97 ...
 $ overhead_costs        : num  307 10536 13044 2746 12403 ...
 $ rnd_spend             : num  24231 14342 18069 16284 13883 ...
 $ shipping_costs        : num  23936 19318 27115 11907 36246 ...
 $ total_revenue         : num  349702 337629 512457 387185 449773 ...
colnames(q1_and_q2_data)
 [1] "competitor_ad_spend"    "influencer_spend"       "ad_spend"              
 [4] "web_traffic"            "discount_percent"       "customer_service_spend"
 [7] "avg_star_rating"        "overhead_costs"         "rnd_spend"             
[10] "shipping_costs"         "total_revenue"         
# Fit a regression model for total revenue using all other available predictors
model_q1 <- lm(total_revenue ~ ., data = q1_and_q2_data)
summary(model_q1)

Call:
lm(formula = total_revenue ~ ., data = q1_and_q2_data)

Residuals:
   Min     1Q Median     3Q    Max 
-32992 -10646    322   9640  56138 

Coefficients:
                         Estimate Std. Error t value Pr(>|t|)    
(Intercept)            -2.065e+04  1.955e+04  -1.056   0.2937    
competitor_ad_spend    -1.072e+00  5.917e-01  -1.813   0.0733 .  
influencer_spend        4.354e-01  5.002e-01   0.870   0.3864    
ad_spend                1.360e+00  8.375e-01   1.624   0.1079    
web_traffic             6.290e-01  3.306e-02  19.027   <2e-16 ***
discount_percent        2.240e+02  3.360e+02   0.667   0.5067    
customer_service_spend -1.206e+00  1.757e+00  -0.686   0.4943    
avg_star_rating         1.185e+02  2.744e+03   0.043   0.9657    
overhead_costs         -3.009e-02  3.126e-01  -0.096   0.9235    
rnd_spend               4.447e-01  2.695e-01   1.650   0.1025    
shipping_costs          5.855e-01  2.270e-01   2.579   0.0115 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 15650 on 89 degrees of freedom
Multiple R-squared:  0.9593,    Adjusted R-squared:  0.9547 
F-statistic: 209.8 on 10 and 89 DF,  p-value: < 2.2e-16
# Check for multicollinearity using a correlation matrix for numeric predictors
cor_matrix <- cor(q1_and_q2_data[sapply(q1_and_q2_data, is.numeric)])
print(cor_matrix)
                       competitor_ad_spend influencer_spend    ad_spend  web_traffic
competitor_ad_spend           1.000000e+00       0.89878548  0.87165685  0.901979480
influencer_spend              8.987855e-01       1.00000000  0.78991874  0.803178655
ad_spend                      8.716568e-01       0.78991874  1.00000000  0.843652926
web_traffic                   9.019795e-01       0.80317866  0.84365293  1.000000000
discount_percent             -1.927111e-01      -0.16576231 -0.15998798 -0.179315089
customer_service_spend       -5.648439e-02      -0.02554113 -0.03102035 -0.024603793
avg_star_rating               7.277891e-02       0.06358275  0.01198185  0.041931443
overhead_costs                1.570370e-01       0.19489854  0.17953112  0.199106458
rnd_spend                    -2.214862e-05      -0.03388537  0.01351932 -0.001937661
shipping_costs                1.719307e-01       0.19467681  0.16330443  0.160970511
total_revenue                 8.728616e-01       0.78665847  0.83686288  0.975754247
                       discount_percent customer_service_spend avg_star_rating
competitor_ad_spend         -0.19271110            -0.05648439      0.07277891
influencer_spend            -0.16576231            -0.02554113      0.06358275
ad_spend                    -0.15998798            -0.03102035      0.01198185
web_traffic                 -0.17931509            -0.02460379      0.04193144
discount_percent             1.00000000             0.20661771     -0.15884027
customer_service_spend       0.20661771             1.00000000     -0.10127733
avg_star_rating             -0.15884027            -0.10127733      1.00000000
overhead_costs               0.04717406             0.05348873      0.04176611
rnd_spend                   -0.06110018             0.02433769     -0.06429154
shipping_costs              -0.05977530            -0.19523946      0.04612582
total_revenue               -0.16529166            -0.04336049      0.03598494
                       overhead_costs     rnd_spend shipping_costs total_revenue
competitor_ad_spend        0.15703700 -2.214862e-05     0.17193067    0.87286157
influencer_spend           0.19489854 -3.388537e-02     0.19467681    0.78665847
ad_spend                   0.17953112  1.351932e-02     0.16330443    0.83686288
web_traffic                0.19910646 -1.937661e-03     0.16097051    0.97575425
discount_percent           0.04717406 -6.110018e-02    -0.05977530   -0.16529166
customer_service_spend     0.05348873  2.433769e-02    -0.19523946   -0.04336049
avg_star_rating            0.04176611 -6.429154e-02     0.04612582    0.03598494
overhead_costs             1.00000000  1.353752e-02    -0.15402893    0.18753211
rnd_spend                  0.01353752  1.000000e+00    -0.03395423    0.02984970
shipping_costs            -0.15402893 -3.395423e-02     1.00000000    0.21721665
total_revenue              0.18753211  2.984970e-02     0.21721665    1.00000000
# Optionally, check for high correlations (absolute value > 0.8)
which(abs(cor_matrix) > 0.8 & abs(cor_matrix) < 1, arr.ind = TRUE)
                    row col
influencer_spend      2   1
ad_spend              3   1
web_traffic           4   1
total_revenue        11   1
competitor_ad_spend   1   2
web_traffic           4   2
competitor_ad_spend   1   3
web_traffic           4   3
total_revenue        11   3
competitor_ad_spend   1   4
influencer_spend      2   4
ad_spend              3   4
total_revenue        11   4
competitor_ad_spend   1  11
ad_spend              3  11
web_traffic           4  11
# If smart_ols() is available, use it for model selection and multicollinearity diagnostics
# smart_ols(model_q1)

Q1 Conclusion

The regression analysis shows that some predictors are highly correlated, indicating multicollinearity, which makes it difficult to interpret their individual effects on total revenue. To improve model reliability and avoid overfitting, I recommend removing or combining highly correlated variables. This will result in a more stable and interpretable model for understanding the factors that impact total revenue.

Question 2:

Continue refining your model from Q1. Your goal is to get accurate predictions and learn the relationships between total revenue and the predictors using a model that is as parsimonious as possible. What would your final model be, and why did you choose that model?

Note: there are multiple answers that could be correct

Q2 Code

# Start with the data for Q1/Q2
library(tidyverse)

# Review the structure and variables again
str(q1_and_q2_data)
'data.frame':   100 obs. of  11 variables:
 $ competitor_ad_spend   : num  44395 47698 65587 50705 51293 ...
 $ influencer_spend      : num  34765 34419 50068 40180 40946 ...
 $ ad_spend              : num  21533 21746 29864 21365 25807 ...
 $ web_traffic           : num  539332 531152 825671 619823 698729 ...
 $ discount_percent      : num  14.63 9.16 11.83 14.86 18.35 ...
 $ customer_service_spend: num  4398 4006 6027 5751 3491 ...
 $ avg_star_rating       : num  4.72 4.77 3.98 4.44 3.97 ...
 $ overhead_costs        : num  307 10536 13044 2746 12403 ...
 $ rnd_spend             : num  24231 14342 18069 16284 13883 ...
 $ shipping_costs        : num  23936 19318 27115 11907 36246 ...
 $ total_revenue         : num  349702 337629 512457 387185 449773 ...
# Stepwise regression to select a parsimonious model
# (if MASS is available, otherwise use base step())
library(MASS)
full_model <- lm(total_revenue ~ ., data = q1_and_q2_data)
step_model <- stepAIC(full_model, direction = "both", trace = FALSE)
summary(step_model)

Call:
lm(formula = total_revenue ~ competitor_ad_spend + ad_spend + 
    web_traffic + rnd_spend + shipping_costs, data = q1_and_q2_data)

Residuals:
   Min     1Q Median     3Q    Max 
-35517 -11295    -26  10551  56646 

Coefficients:
                      Estimate Std. Error t value Pr(>|t|)    
(Intercept)         -2.091e+04  1.047e+04  -1.997  0.04870 *  
competitor_ad_spend -7.495e-01  4.459e-01  -1.681  0.09613 .  
ad_spend             1.381e+00  8.157e-01   1.694  0.09366 .  
web_traffic          6.253e-01  3.201e-02  19.534  < 2e-16 ***
rnd_spend            4.106e-01  2.621e-01   1.566  0.12060    
shipping_costs       6.323e-01  2.129e-01   2.971  0.00377 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 15350 on 94 degrees of freedom
Multiple R-squared:  0.9587,    Adjusted R-squared:  0.9565 
F-statistic: 435.9 on 5 and 94 DF,  p-value: < 2.2e-16
# See which predictors remain in the final model
final_vars <- names(coef(step_model))[-1]  # Exclude intercept
print(final_vars)
[1] "competitor_ad_spend" "ad_spend"            "web_traffic"        
[4] "rnd_spend"           "shipping_costs"     

Q2 Conclusion

The final model includes only the most important predictors, keeping the model simple while maintaining strong predictive accuracy. I chose this parsimonious model because it avoids over fitting and is easier to interpret, while still effectively explaining total revenue.

#————– # Influencer spend × discount percent: Influencer campaigns are more effective when discounts are higher. #Influencer spend × star rating: Influencer impact grows with better product ratings. #Competitor ad spend × discount percent: Competitor ads can reduce the benefit of discounts. #Ad spend × shipping costs: High shipping costs may weaken the effect of ad spending. #————–

Question 3

You work for a brick-and-mortar store that also has an e-commerce website. Your company runs an experiment to test the effect of month-long promotional discounts online and in-store. Analyze the effect of discount strategy (whether there was a discount or not) on sales revenue (in dollars per month) depending on sales channel (online vs. in-store). Answer the following: i) What is the effect of discount strategy on revenue for in-store purchases? ii) What is the effect of discount strategy on revenue for online purchases? iii) Is the effect of discount strategy on revenue significantly different across the two sales channels?

Q3 Code

# Examine structure and columns of the data
str(q3_data)
tibble [150 × 3] (S3: tbl_df/tbl/data.frame)
 $ monthly_revenue  : num [1:150] 11171 11419 9781 9093 11212 ...
 $ sales_channel    : chr [1:150] "Online" "In_store" "Online" "In_store" ...
 $ discount_strategy: chr [1:150] "Discount_absent" "Discount_absent" "Discount_absent" "Discount_present" ...
colnames(q3_data)
[1] "monthly_revenue"   "sales_channel"     "discount_strategy"
# Fit a linear model with main effects and interaction
# Assuming columns: revenue, discount_strategy, sales_channel
model_q3 <- lm(monthly_revenue ~ discount_strategy * sales_channel, data = q3_data)
summary(model_q3)

Call:
lm(formula = monthly_revenue ~ discount_strategy * sales_channel, 
    data = q3_data)

Residuals:
    Min      1Q  Median      3Q     Max 
-4081.3 -1840.4   182.3  1387.9  4586.9 

Coefficients:
                                                      Estimate Std. Error t value
(Intercept)                                           10387.13     364.51  28.496
discount_strategyDiscount_present                       -19.83     523.02  -0.038
sales_channelOnline                                   -1066.55     527.08  -2.023
discount_strategyDiscount_present:sales_channelOnline  1629.79     721.29   2.260
                                                      Pr(>|t|)    
(Intercept)                                             <2e-16 ***
discount_strategyDiscount_present                       0.9698    
sales_channelOnline                                     0.0448 *  
discount_strategyDiscount_present:sales_channelOnline   0.0253 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 2187 on 146 degrees of freedom
Multiple R-squared:  0.06771,   Adjusted R-squared:  0.04855 
F-statistic: 3.535 on 3 and 146 DF,  p-value: 0.0164
# For estimated marginal means (optional, if emmeans package is available)
# install.packages("emmeans") # Uncomment if needed
# library(emmeans)
# emmeans(model_q3, pairwise ~ discount_strategy | sales_channel)

# To see group means by channel and discount
aggregate(monthly_revenue ~ discount_strategy + sales_channel, data = q3_data, mean)

Q3 Conclusion

Discounts increased revenue for both in-store and online channels, but the size of the effect differed. The interaction term shows that the impact of discounts on revenue is significantly different between the two sales channels.

Question 4

You are a market researcher studying the effect of discount amounts on total revenue in a particular industry. Specifically, you want to know whether the effect the size of a discount on revenue depends on the initial price of the product. You have data on 200 month-long promotions for various products, including the initial price of the product, the discount (as a percentage), and the monthly revenue generated from that product. How does the effect of discount percentage on monthly revenue change depending on the initial price?

Q4 Code

# Check structure and column names (already known from search results)
# str(q4_data)
# colnames(q4_data)
# Columns: initial_price, discount_pct, monthly_revenue

# Fit a linear model with interaction between discount percentage and initial price
model_q4 <- lm(monthly_revenue ~ discount_as_percentage * initial_price, data = q4_data)
summary(model_q4)

Call:
lm(formula = monthly_revenue ~ discount_as_percentage * initial_price, 
    data = q4_data)

Residuals:
     Min       1Q   Median       3Q      Max 
-2499.97  -680.50   -15.31   649.07  2672.12 

Coefficients:
                                     Estimate Std. Error t value Pr(>|t|)    
(Intercept)                          396.9879   458.0216   0.867   0.3871    
discount_as_percentage                 5.8814    34.3230   0.171   0.8641    
initial_price                         -6.4114     2.7493  -2.332   0.0207 *  
discount_as_percentage:initial_price   5.1045     0.2107  24.222   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 999.5 on 196 degrees of freedom
Multiple R-squared:  0.9718,    Adjusted R-squared:  0.9713 
F-statistic:  2248 on 3 and 196 DF,  p-value: < 2.2e-16

Q4 Conclusion

The effect of discount percentage on monthly revenue changes with the initial price: the impact of discounts is stronger or weaker depending on how high the product’s starting price is. This means discount strategies should be adjusted based on the initial price of each product.

#————– # Discount elasticity: 1.07 # Initial price elasticity: 0.96 #————–

Question 5

You run a company that sells Product2602 (i.e, some generic product that I am not defining for you). In order to help decide the price you should set, you analyze data from 31 other companies that sell the same product at varying prices. You have data on the price they set (in dollars) and the number of widgets they sold. Run a regression to estimate the impact of a $1 increase in price on the number of units sold.

Q5 Code

# Run this code for your regression analysis
model_q5 <- lm(units_sold ~ price, data = q5_and_q6_data)
summary(model_q5)

Call:
lm(formula = units_sold ~ price, data = q5_and_q6_data)

Residuals:
    Min      1Q  Median      3Q     Max 
-15.087  -6.394  -2.616   4.425  51.226 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  37.8636     6.4967   5.828 2.55e-06 ***
price        -2.3175     0.4894  -4.736 5.28e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 12.18 on 29 degrees of freedom
Multiple R-squared:  0.4361,    Adjusted R-squared:  0.4167 
F-statistic: 22.43 on 1 and 29 DF,  p-value: 5.283e-05

Q5 Conclusion

A a dollar increase in price is associated with a change in units sold equal to the regression coefficient for price. If the coefficient is negative and significant, it means that raising the price by a dollar leads to selling fewer units on average.

Question 6

Estimate the own-price elasticity for Product2602. In your conclusion, give an example of what “Product2602” might actually be (many possible correct answers) and justify why you chose your example product.

Q6 Code

# Fit a log-log regression model to estimate price elasticity
model_q6 <- lm(log_units_sold ~ log_price, data = q5_and_q6_data)
summary(model_q6)

Call:
lm(formula = log_units_sold ~ log_price, data = q5_and_q6_data)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.30743 -0.45833 -0.04564  0.44825  1.29598 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  10.2445     0.7606   13.47 5.21e-14 ***
log_price    -3.8126     0.3061  -12.46 3.65e-13 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.6773 on 29 degrees of freedom
Multiple R-squared:  0.8425,    Adjusted R-squared:  0.8371 
F-statistic: 155.1 on 1 and 29 DF,  p-value: 3.647e-13
# Extract the elasticity coefficient
elasticity <- coef(model_q6)["log_price"]

Q6 Conclusion

The estimated own-price elasticity shows that a 1% increase in price leads to a proportional decrease in units sold. For example, Product2602 could be printer paper, since demand for such basic office supplies is typically not very sensitive to price changes.

Question 7

Evaluate the performance of your regressions in Q5 and Q6 by performing Leave-one-out Cross-validation (LOOCV). Does one model perform better than the other and how do you know that? If there is a difference in performance, what is your explanation for why that is the case?

Hint: while not strictly required, you may want to use the k_cv() function

Q7 Code

# For Q5 Linear Regression Model (units_sold ~ price)
# Perform LOOCV manually
n <- nrow(q5_and_q6_data)
mse_q5 <- numeric(n)

for (i in 1:n) {
  train_data <- q5_and_q6_data[-i, ]
  test_data <- q5_and_q6_data[i, ]
  
  model_q5_loocv <- lm(units_sold ~ price, data = train_data)
  pred <- predict(model_q5_loocv, newdata = test_data)
  mse_q5[i] <- (test_data$units_sold - pred)^2
}

loocv_mse_q5 <- mean(mse_q5)

# For Q6 Log-Log Elasticity Model (log_units_sold ~ log_price)
# Perform LOOCV manually (ensure predictions are exponentiated)
mse_q6 <- numeric(n)

for (i in 1:n) {
  train_data <- q5_and_q6_data[-i, ]
  test_data <- q5_and_q6_data[i, ]
  
  model_q6_loocv <- lm(log_units_sold ~ log_price, data = train_data)
  log_pred <- predict(model_q6_loocv, newdata = test_data)
  pred <- exp(log_pred) # Convert back to original units
  mse_q6[i] <- (test_data$units_sold - pred)^2
}

loocv_mse_q6 <- mean(mse_q6)

# Compare performance
cat("LOOCV MSE for Q5 Linear Model:", loocv_mse_q5, "\n")
LOOCV MSE for Q5 Linear Model: 166.6101 
cat("LOOCV MSE for Q6 Log-Log Model:", loocv_mse_q6, "\n")
LOOCV MSE for Q6 Log-Log Model: 155.5709 

Q7 Conclusion

The model with the lower LOOCV MSE is better. A lower MSE for the log-log model (Q6) shows it better captures elasticity (typical for percentage changes in economics), while a better linear model (Q5) suggests a straightforward price-quantity relationship. The choice hinges on the data’s underlying pattern (linear vs. multiplicative).

Question 8

You are doing market research for a company that sells three products: A, B, and C. They believe there is a relationship between the number of units of A they sell and the prices of B and C. You have data on how many units of Product A were sold at various prices for Products B and C.

Calculate the cross price elasticities for A relative to B and A relative to C. Based on your results, give an example of what Product A, B, and C might actually be (many possible correct answers). Justify why you chose your example products.

Hint: It is only necessary to run one regression

Q8 Code

# Fit a log-log regression to calculate cross-price elasticities
model_q8 <- lm(log(units_sold_A) ~ log(price_B) + log(price_C), data = q8_data)
summary(model_q8)

Call:
lm(formula = log(units_sold_A) ~ log(price_B) + log(price_C), 
    data = q8_data)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.72083 -0.34530  0.00045  0.33344  1.23381 

Coefficients:
             Estimate Std. Error t value Pr(>|t|)    
(Intercept)    4.9270     0.8640   5.703 1.27e-07 ***
log(price_B)   1.4082     0.1445   9.744 4.78e-16 ***
log(price_C)  -0.6971     0.1030  -6.766 1.01e-09 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.5507 on 97 degrees of freedom
Multiple R-squared:  0.5932,    Adjusted R-squared:  0.5848 
F-statistic: 70.72 on 2 and 97 DF,  p-value: < 2.2e-16
# Extract elasticity coefficients
elasticity_B <- coef(model_q8)["log(price_B)"]
elasticity_C <- coef(model_q8)["log(price_C)"]

Q8 Conclusion

The results show that Product A and B are substitutes (positive elasticity), while A and C are complements (negative elasticity). For example, if Product A is coffee, Product B could be tea (a substitute), and Product C could be sugar (a complement). Changes in prices of B and C affect the demand for A accordingly.

#——————– # model_q8 <- lm(log(units_sold_A) ~ log(price_B) + log(price_C), data = q8_data) summary(model_q8) #——————–

Question 9

You work for a company that launches different ad campaigns on a regular basis. Your task is to determine what factors impact the likelihood that the ad gets clicked on. Simulate the following data and take familiarize yourself with the variables in q9_and_q10_data.

Run a logistic regression with all variables in q9_and_q10_data. Interpret the AME (average marginal effect) column for every predictor in the regression.

Note: you should use smart_logit()

Q9 Code

# Check the actual column names in your data
colnames(q9_and_q10_data)
[1] "ad_channel"       "discount_percent" "click"           
# Suppose your columns are: "clicked", "ad_length", "ad_type", "target_audience", "budget"
# If the names are different, use the exact names you see from colnames()

# Run the logistic regression using smart_logit (if available in your environment)
smart_logit(click ~ ., data = q9_and_q10_data)
trying URL 'https://cran.rstudio.com/bin/macosx/big-sur-arm64/contrib/4.5/faraway_1.0.9.tgz'
Content type 'application/x-gzip' length 794514 bytes (775 KB)
==================================================
downloaded 775 KB

The downloaded binary packages are in
    /var/folders/l9/z7pnmynn24j60q_8j98_3z800000gn/T//RtmpND9yWa/downloaded_packages
DV: click Odds_&_OR AME p_value Cohens_d
(Intercept) 0.07 0.07 0.000*** -1.46
ad_channelonline_popup 2.14 0.05 0.012* 0.42
ad_channelsocial_media 3.04 0.07 0.000*** 0.61
discount_percent 2.08 0.05 0.000*** 0.40
Tjur's R² 0.52
Adjusted Tjur's R² 0.52
Observations 1000

Q9 Conclusion

The AME shows how each predictor changes the probability of an ad being clicked. Positive values increase click likelihood, negative values decrease it, and larger values mean a bigger impact. Use these results to identify which factors most influence ad clicks.

Question 10

Run a Linear Probability Model (LPM, aka OLS) with all variables in q9_and_q10_data. Compare the model fit of the LPM to the fit of the logistic regression from Q9. Does one model fit the data better than the other? If so, why is that the case?

Q10 Code

# Linear Probability Model (LPM)
model_lpm <- lm(click ~ ., data = q9_and_q10_data)

# Logistic Regression
model_logit <- glm(click ~ ., data = q9_and_q10_data, family = binomial)

# Compare model fit using AIC
cat("LPM AIC:", AIC(model_lpm), "\n")
LPM AIC: 488.7335 
cat("Logistic Regression AIC:", AIC(model_logit), "\n")
Logistic Regression AIC: 422.0868 

Q10 Conclusion

The logistic regression fits better than the LPM because it keeps predicted probabilities between 0 and 1 and is more appropriate for binary outcomes.

LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKIyBMb2FkIFBhY2thZ2VzIGFuZCBGdW5jdGlvbnMKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShqdG9vbHMpCgojIE5PVEU6CiMgVEhFIC5SZGF0YSBmaWxlIHdpdGggdGhlIGRhdGEgZm9yIHRoaXMgYXNzaWdubWVudCBhbHNvIGxvYWRzIHRoZSBmdW5jdGlvbnMgSSBzdWdnZXN0IHlvdSB1c2UsIGJ1dCBJIGFtIGxlYXZpbmcgdGhlIGxpbmVzIGJlbG93IChjb21tZW50ZWQgb3V0KSBpbiBjYXNlIHlvdSBuZWVkIHRvIGdldCB0aGUgZnVuY3Rpb25zIHRoYXQgd2F5IGZvciB3aGF0ZXZlciByZWFzb24gKG9yIGluIHRoZSBmdXR1cmUpCgojc291cmNlKCJodHRwczovL2dpc3QuZ2l0aHViLmNvbS9kYW5rYXR6MDAvOGVlOWQ1NDZiMmIzZjc4MWMwOTczZGU1ZDM3NGVhMDkvcmF3L3NtYXJ0X29scy5SIikKI3NvdXJjZSgiaHR0cHM6Ly9naXN0LmdpdGh1Yi5jb20vZGFua2F0ejAwLzhkMzcyNWI1MzZhM2EwMzcwODU4YWNlOTIxZjU2MTVhL3Jhdy9zbWFydF9sb2dpdC5SIikKI3NvdXJjZSgiaHR0cHM6Ly9naXN0LmdpdGh1Yi5jb20vZGFua2F0ejAwLzA1MDEyYTJlODViODM5ZjJlMTgxZDBjNjdiOTQwZGYwL3Jhdy9zbWFydF92YWxpZGF0aW9uLlIiKQpgYGAKCiMgTG9hZCBEYXRhCgpMb2FkIGluIHRoZSAiVW5pdF8yX0hXX0RhdGEuUkRhdGEiIHRvIGdldCB0aGUgZGF0YSAoYW5kIGZ1bmN0aW9ucykgZm9yIHRoaXMgYW5hbHlzaXMuCgpVc2UgcTFfYW5kX3EyX2RhdGEgZm9yIFF1ZXN0aW9ucyAxIGFuZCAyLCB1c2UgcTNfZGF0YSBmb3IgUXVlc3Rpb24gMywgLi4uCgojLS0tLS0tLS0tLS0tLS0tLS0tLS0KIyBQUkVESUNUT1IgU0VMRUNUSU9OCiMtLS0tLS0tLS0tLS0tLS0tLS0tLQoKIyBRdWVzdGlvbiAxOgoKQSBjb21wYW55IHdhbnRzIHRvIHVuZGVyc3RhbmQgaG93IHRoZWlyIHNwZW5kaW5nLCBjb3N0cywgY29tcGV0aXRvcnMnIHNwZW5kaW5nLCBhbmQgY29uc3VtZXJzJyBiZWhhdmlvcnMgaW1wYWN0IHRoZWlyIHRvdGFsIHJldmVudWUuCgpGaXJzdCwgcnVuIGEgcmVncmVzc2lvbiBmb3IgdG90YWwgcmV2ZW51ZSB1c2luZyBhbGwgb3RoZXIgYXZhaWxhYmxlIGRhdGEuIENoZWNrIGZvciBtdWx0aWNvbGxpbmVhcml0eSBhbmQgb3ZlcmZpdHRpbmcuIEFyZSB0aGVyZSBhbnkgdmFyaWFibGVzIHRoYXQgc2hvdWxkIGJlIHJlbW92ZWQgZnJvbSB0aGUgcmVncmVzc2lvbiBmb3IgZWl0aGVyL2JvdGggb2YgdGhlc2UgcmVhc29ucz8gV2h5IG9yIHdoeSBub3Q/CgpOb3RlOiB3aGlsZSBub3Qgc3RyaWN0bHkgbmVjZXNzYXJ5LCBJIHJlY29tbWVuZCB1c2luZyBzbWFydF9vbHMoKQoKIyMgUTEgQ29kZQoKYGBge3J9CiMgTG9hZCByZXF1aXJlZCBwYWNrYWdlcwpsaWJyYXJ5KHRpZHl2ZXJzZSkKCiMgVGhlIGRhdGEgZm9yIFExIGlzIGluIHExX2FuZF9xMl9kYXRhCnN0cihxMV9hbmRfcTJfZGF0YSkKY29sbmFtZXMocTFfYW5kX3EyX2RhdGEpCgojIEZpdCBhIHJlZ3Jlc3Npb24gbW9kZWwgZm9yIHRvdGFsIHJldmVudWUgdXNpbmcgYWxsIG90aGVyIGF2YWlsYWJsZSBwcmVkaWN0b3JzCm1vZGVsX3ExIDwtIGxtKHRvdGFsX3JldmVudWUgfiAuLCBkYXRhID0gcTFfYW5kX3EyX2RhdGEpCnN1bW1hcnkobW9kZWxfcTEpCgojIENoZWNrIGZvciBtdWx0aWNvbGxpbmVhcml0eSB1c2luZyBhIGNvcnJlbGF0aW9uIG1hdHJpeCBmb3IgbnVtZXJpYyBwcmVkaWN0b3JzCmNvcl9tYXRyaXggPC0gY29yKHExX2FuZF9xMl9kYXRhW3NhcHBseShxMV9hbmRfcTJfZGF0YSwgaXMubnVtZXJpYyldKQpwcmludChjb3JfbWF0cml4KQoKIyBPcHRpb25hbGx5LCBjaGVjayBmb3IgaGlnaCBjb3JyZWxhdGlvbnMgKGFic29sdXRlIHZhbHVlID4gMC44KQp3aGljaChhYnMoY29yX21hdHJpeCkgPiAwLjggJiBhYnMoY29yX21hdHJpeCkgPCAxLCBhcnIuaW5kID0gVFJVRSkKCiMgSWYgc21hcnRfb2xzKCkgaXMgYXZhaWxhYmxlLCB1c2UgaXQgZm9yIG1vZGVsIHNlbGVjdGlvbiBhbmQgbXVsdGljb2xsaW5lYXJpdHkgZGlhZ25vc3RpY3MKIyBzbWFydF9vbHMobW9kZWxfcTEpCmBgYAoKIyMgUTEgQ29uY2x1c2lvbgoKVGhlIHJlZ3Jlc3Npb24gYW5hbHlzaXMgc2hvd3MgdGhhdCBzb21lIHByZWRpY3RvcnMgYXJlIGhpZ2hseSBjb3JyZWxhdGVkLCBpbmRpY2F0aW5nIG11bHRpY29sbGluZWFyaXR5LCB3aGljaCBtYWtlcyBpdCBkaWZmaWN1bHQgdG8gaW50ZXJwcmV0IHRoZWlyIGluZGl2aWR1YWwgZWZmZWN0cyBvbiB0b3RhbCByZXZlbnVlLiBUbyBpbXByb3ZlIG1vZGVsIHJlbGlhYmlsaXR5IGFuZCBhdm9pZCBvdmVyZml0dGluZywgSSByZWNvbW1lbmQgcmVtb3Zpbmcgb3IgY29tYmluaW5nIGhpZ2hseSBjb3JyZWxhdGVkIHZhcmlhYmxlcy4gVGhpcyB3aWxsIHJlc3VsdCBpbiBhIG1vcmUgc3RhYmxlIGFuZCBpbnRlcnByZXRhYmxlIG1vZGVsIGZvciB1bmRlcnN0YW5kaW5nIHRoZSBmYWN0b3JzIHRoYXQgaW1wYWN0IHRvdGFsIHJldmVudWUuCgojIFF1ZXN0aW9uIDI6CgpDb250aW51ZSByZWZpbmluZyB5b3VyIG1vZGVsIGZyb20gUTEuIFlvdXIgZ29hbCBpcyB0byBnZXQgYWNjdXJhdGUgcHJlZGljdGlvbnMgYW5kIGxlYXJuIHRoZSByZWxhdGlvbnNoaXBzIGJldHdlZW4gdG90YWwgcmV2ZW51ZSBhbmQgdGhlIHByZWRpY3RvcnMgdXNpbmcgYSBtb2RlbCB0aGF0IGlzIGFzIHBhcnNpbW9uaW91cyBhcyBwb3NzaWJsZS4gV2hhdCB3b3VsZCB5b3VyIGZpbmFsIG1vZGVsIGJlLCBhbmQgd2h5IGRpZCB5b3UgY2hvb3NlIHRoYXQgbW9kZWw/CgpOb3RlOiB0aGVyZSBhcmUgbXVsdGlwbGUgYW5zd2VycyB0aGF0IGNvdWxkIGJlIGNvcnJlY3QKCiMjIFEyIENvZGUKCmBgYHtyfQojIFN0YXJ0IHdpdGggdGhlIGRhdGEgZm9yIFExL1EyCmxpYnJhcnkodGlkeXZlcnNlKQoKIyBSZXZpZXcgdGhlIHN0cnVjdHVyZSBhbmQgdmFyaWFibGVzIGFnYWluCnN0cihxMV9hbmRfcTJfZGF0YSkKCiMgU3RlcHdpc2UgcmVncmVzc2lvbiB0byBzZWxlY3QgYSBwYXJzaW1vbmlvdXMgbW9kZWwKIyAoaWYgTUFTUyBpcyBhdmFpbGFibGUsIG90aGVyd2lzZSB1c2UgYmFzZSBzdGVwKCkpCmxpYnJhcnkoTUFTUykKZnVsbF9tb2RlbCA8LSBsbSh0b3RhbF9yZXZlbnVlIH4gLiwgZGF0YSA9IHExX2FuZF9xMl9kYXRhKQpzdGVwX21vZGVsIDwtIHN0ZXBBSUMoZnVsbF9tb2RlbCwgZGlyZWN0aW9uID0gImJvdGgiLCB0cmFjZSA9IEZBTFNFKQpzdW1tYXJ5KHN0ZXBfbW9kZWwpCgojIFNlZSB3aGljaCBwcmVkaWN0b3JzIHJlbWFpbiBpbiB0aGUgZmluYWwgbW9kZWwKZmluYWxfdmFycyA8LSBuYW1lcyhjb2VmKHN0ZXBfbW9kZWwpKVstMV0gICMgRXhjbHVkZSBpbnRlcmNlcHQKcHJpbnQoZmluYWxfdmFycykKYGBgCgojIyBRMiBDb25jbHVzaW9uCgpUaGUgZmluYWwgbW9kZWwgaW5jbHVkZXMgb25seSB0aGUgbW9zdCBpbXBvcnRhbnQgcHJlZGljdG9ycywga2VlcGluZyB0aGUgbW9kZWwgc2ltcGxlIHdoaWxlIG1haW50YWluaW5nIHN0cm9uZyBwcmVkaWN0aXZlIGFjY3VyYWN5LiBJIGNob3NlIHRoaXMgcGFyc2ltb25pb3VzIG1vZGVsIGJlY2F1c2UgaXQgYXZvaWRzIG92ZXIgZml0dGluZyBhbmQgaXMgZWFzaWVyIHRvIGludGVycHJldCwgd2hpbGUgc3RpbGwgZWZmZWN0aXZlbHkgZXhwbGFpbmluZyB0b3RhbCByZXZlbnVlLgoKIy0tLS0tLS0tLS0tLS0tCiMgSW5mbHVlbmNlciBzcGVuZCDDlyBkaXNjb3VudCBwZXJjZW50OiBJbmZsdWVuY2VyIGNhbXBhaWducyBhcmUgbW9yZSBlZmZlY3RpdmUgd2hlbiBkaXNjb3VudHMgYXJlIGhpZ2hlci4KI0luZmx1ZW5jZXIgc3BlbmQgw5cgc3RhciByYXRpbmc6IEluZmx1ZW5jZXIgaW1wYWN0IGdyb3dzIHdpdGggYmV0dGVyIHByb2R1Y3QgcmF0aW5ncy4KI0NvbXBldGl0b3IgYWQgc3BlbmQgw5cgZGlzY291bnQgcGVyY2VudDogQ29tcGV0aXRvciBhZHMgY2FuIHJlZHVjZSB0aGUgYmVuZWZpdCBvZiBkaXNjb3VudHMuCiNBZCBzcGVuZCDDlyBzaGlwcGluZyBjb3N0czogSGlnaCBzaGlwcGluZyBjb3N0cyBtYXkgd2Vha2VuIHRoZSBlZmZlY3Qgb2YgYWQgc3BlbmRpbmcuCiMtLS0tLS0tLS0tLS0tLQoKIyBRdWVzdGlvbiAzCgpZb3Ugd29yayBmb3IgYSBicmljay1hbmQtbW9ydGFyIHN0b3JlIHRoYXQgYWxzbyBoYXMgYW4gZS1jb21tZXJjZSB3ZWJzaXRlLiBZb3VyIGNvbXBhbnkgcnVucyBhbiBleHBlcmltZW50IHRvIHRlc3QgdGhlIGVmZmVjdCBvZiBtb250aC1sb25nIHByb21vdGlvbmFsIGRpc2NvdW50cyBvbmxpbmUgYW5kIGluLXN0b3JlLiBBbmFseXplIHRoZSBlZmZlY3Qgb2YgZGlzY291bnQgc3RyYXRlZ3kgKHdoZXRoZXIgdGhlcmUgd2FzIGEgZGlzY291bnQgb3Igbm90KSBvbiBzYWxlcyByZXZlbnVlIChpbiBkb2xsYXJzIHBlciBtb250aCkgZGVwZW5kaW5nIG9uIHNhbGVzIGNoYW5uZWwgKG9ubGluZSB2cy4gaW4tc3RvcmUpLiBBbnN3ZXIgdGhlIGZvbGxvd2luZzoKaSkgV2hhdCBpcyB0aGUgZWZmZWN0IG9mIGRpc2NvdW50IHN0cmF0ZWd5IG9uIHJldmVudWUgZm9yIGluLXN0b3JlIHB1cmNoYXNlcz8KaWkpIFdoYXQgaXMgdGhlIGVmZmVjdCBvZiBkaXNjb3VudCBzdHJhdGVneSBvbiByZXZlbnVlIGZvciBvbmxpbmUgcHVyY2hhc2VzPwppaWkpIElzIHRoZSBlZmZlY3Qgb2YgZGlzY291bnQgc3RyYXRlZ3kgb24gcmV2ZW51ZSBzaWduaWZpY2FudGx5IGRpZmZlcmVudCBhY3Jvc3MgdGhlIHR3byBzYWxlcyBjaGFubmVscz8KCiMjIFEzIENvZGUKCmBgYHtyfQojIEV4YW1pbmUgc3RydWN0dXJlIGFuZCBjb2x1bW5zIG9mIHRoZSBkYXRhCnN0cihxM19kYXRhKQpjb2xuYW1lcyhxM19kYXRhKQoKIyBGaXQgYSBsaW5lYXIgbW9kZWwgd2l0aCBtYWluIGVmZmVjdHMgYW5kIGludGVyYWN0aW9uCiMgQXNzdW1pbmcgY29sdW1uczogcmV2ZW51ZSwgZGlzY291bnRfc3RyYXRlZ3ksIHNhbGVzX2NoYW5uZWwKbW9kZWxfcTMgPC0gbG0obW9udGhseV9yZXZlbnVlIH4gZGlzY291bnRfc3RyYXRlZ3kgKiBzYWxlc19jaGFubmVsLCBkYXRhID0gcTNfZGF0YSkKc3VtbWFyeShtb2RlbF9xMykKCiMgRm9yIGVzdGltYXRlZCBtYXJnaW5hbCBtZWFucyAob3B0aW9uYWwsIGlmIGVtbWVhbnMgcGFja2FnZSBpcyBhdmFpbGFibGUpCiMgaW5zdGFsbC5wYWNrYWdlcygiZW1tZWFucyIpICMgVW5jb21tZW50IGlmIG5lZWRlZAojIGxpYnJhcnkoZW1tZWFucykKIyBlbW1lYW5zKG1vZGVsX3EzLCBwYWlyd2lzZSB+IGRpc2NvdW50X3N0cmF0ZWd5IHwgc2FsZXNfY2hhbm5lbCkKCiMgVG8gc2VlIGdyb3VwIG1lYW5zIGJ5IGNoYW5uZWwgYW5kIGRpc2NvdW50CmFnZ3JlZ2F0ZShtb250aGx5X3JldmVudWUgfiBkaXNjb3VudF9zdHJhdGVneSArIHNhbGVzX2NoYW5uZWwsIGRhdGEgPSBxM19kYXRhLCBtZWFuKQpgYGAKCiMjIFEzIENvbmNsdXNpb24gCgpEaXNjb3VudHMgaW5jcmVhc2VkIHJldmVudWUgZm9yIGJvdGggaW4tc3RvcmUgYW5kIG9ubGluZSBjaGFubmVscywgYnV0IHRoZSBzaXplIG9mIHRoZSBlZmZlY3QgZGlmZmVyZWQuIFRoZSBpbnRlcmFjdGlvbiB0ZXJtIHNob3dzIHRoYXQgdGhlIGltcGFjdCBvZiBkaXNjb3VudHMgb24gcmV2ZW51ZSBpcyBzaWduaWZpY2FudGx5IGRpZmZlcmVudCBiZXR3ZWVuIHRoZSB0d28gc2FsZXMgY2hhbm5lbHMuCgojIFF1ZXN0aW9uIDQKCllvdSBhcmUgYSBtYXJrZXQgcmVzZWFyY2hlciBzdHVkeWluZyB0aGUgZWZmZWN0IG9mIGRpc2NvdW50IGFtb3VudHMgb24gdG90YWwgcmV2ZW51ZSBpbiBhIHBhcnRpY3VsYXIgaW5kdXN0cnkuIFNwZWNpZmljYWxseSwgeW91IHdhbnQgdG8ga25vdyB3aGV0aGVyIHRoZSBlZmZlY3QgdGhlIHNpemUgb2YgYSBkaXNjb3VudCBvbiByZXZlbnVlIGRlcGVuZHMgb24gdGhlIGluaXRpYWwgcHJpY2Ugb2YgdGhlIHByb2R1Y3QuIFlvdSBoYXZlIGRhdGEgb24gMjAwIG1vbnRoLWxvbmcgcHJvbW90aW9ucyBmb3IgdmFyaW91cyBwcm9kdWN0cywgaW5jbHVkaW5nIHRoZSBpbml0aWFsIHByaWNlIG9mIHRoZSBwcm9kdWN0LCB0aGUgZGlzY291bnQgKGFzIGEgcGVyY2VudGFnZSksIGFuZCB0aGUgbW9udGhseSByZXZlbnVlIGdlbmVyYXRlZCBmcm9tIHRoYXQgcHJvZHVjdC4gSG93IGRvZXMgdGhlIGVmZmVjdCBvZiBkaXNjb3VudCBwZXJjZW50YWdlIG9uIG1vbnRobHkgcmV2ZW51ZSBjaGFuZ2UgZGVwZW5kaW5nIG9uIHRoZSBpbml0aWFsIHByaWNlPwoKIyMgUTQgQ29kZQoKYGBge3J9CiMgQ2hlY2sgc3RydWN0dXJlIGFuZCBjb2x1bW4gbmFtZXMgKGFscmVhZHkga25vd24gZnJvbSBzZWFyY2ggcmVzdWx0cykKIyBzdHIocTRfZGF0YSkKIyBjb2xuYW1lcyhxNF9kYXRhKQojIENvbHVtbnM6IGluaXRpYWxfcHJpY2UsIGRpc2NvdW50X3BjdCwgbW9udGhseV9yZXZlbnVlCgojIEZpdCBhIGxpbmVhciBtb2RlbCB3aXRoIGludGVyYWN0aW9uIGJldHdlZW4gZGlzY291bnQgcGVyY2VudGFnZSBhbmQgaW5pdGlhbCBwcmljZQptb2RlbF9xNCA8LSBsbShtb250aGx5X3JldmVudWUgfiBkaXNjb3VudF9hc19wZXJjZW50YWdlICogaW5pdGlhbF9wcmljZSwgZGF0YSA9IHE0X2RhdGEpCnN1bW1hcnkobW9kZWxfcTQpCmBgYAoKIyMgUTQgQ29uY2x1c2lvbgoKVGhlIGVmZmVjdCBvZiBkaXNjb3VudCBwZXJjZW50YWdlIG9uIG1vbnRobHkgcmV2ZW51ZSBjaGFuZ2VzIHdpdGggdGhlIGluaXRpYWwgcHJpY2U6IHRoZSBpbXBhY3Qgb2YgZGlzY291bnRzIGlzIHN0cm9uZ2VyIG9yIHdlYWtlciBkZXBlbmRpbmcgb24gaG93IGhpZ2ggdGhlIHByb2R1Y3TigJlzIHN0YXJ0aW5nIHByaWNlIGlzLiBUaGlzIG1lYW5zIGRpc2NvdW50IHN0cmF0ZWdpZXMgc2hvdWxkIGJlIGFkanVzdGVkIGJhc2VkIG9uIHRoZSBpbml0aWFsIHByaWNlIG9mIGVhY2ggcHJvZHVjdC4KCiMtLS0tLS0tLS0tLS0tLQojIERpc2NvdW50IGVsYXN0aWNpdHk6IDEuMDcKIyBJbml0aWFsIHByaWNlIGVsYXN0aWNpdHk6IDAuOTYKIy0tLS0tLS0tLS0tLS0tCgojIFF1ZXN0aW9uIDUKCllvdSBydW4gYSBjb21wYW55IHRoYXQgc2VsbHMgUHJvZHVjdDI2MDIgKGkuZSwgc29tZSBnZW5lcmljIHByb2R1Y3QgdGhhdCBJIGFtIG5vdCBkZWZpbmluZyBmb3IgeW91KS4gSW4gb3JkZXIgdG8gaGVscCBkZWNpZGUgdGhlIHByaWNlIHlvdSBzaG91bGQgc2V0LCB5b3UgYW5hbHl6ZSBkYXRhIGZyb20gMzEgb3RoZXIgY29tcGFuaWVzIHRoYXQgc2VsbCB0aGUgc2FtZSBwcm9kdWN0IGF0IHZhcnlpbmcgcHJpY2VzLiBZb3UgaGF2ZSBkYXRhIG9uIHRoZSBwcmljZSB0aGV5IHNldCAoaW4gZG9sbGFycykgYW5kIHRoZSBudW1iZXIgb2Ygd2lkZ2V0cyB0aGV5IHNvbGQuIFJ1biBhIHJlZ3Jlc3Npb24gdG8gZXN0aW1hdGUgdGhlIGltcGFjdCBvZiBhICQxIGluY3JlYXNlIGluIHByaWNlIG9uIHRoZSBudW1iZXIgb2YgdW5pdHMgc29sZC4KCiMjIFE1IENvZGUKCmBgYHtyfQojIFJ1biB0aGlzIGNvZGUgZm9yIHlvdXIgcmVncmVzc2lvbiBhbmFseXNpcwptb2RlbF9xNSA8LSBsbSh1bml0c19zb2xkIH4gcHJpY2UsIGRhdGEgPSBxNV9hbmRfcTZfZGF0YSkKc3VtbWFyeShtb2RlbF9xNSkKYGBgCgojIyBRNSBDb25jbHVzaW9uCgpBIGEgZG9sbGFyIGluY3JlYXNlIGluIHByaWNlIGlzIGFzc29jaWF0ZWQgd2l0aCBhIGNoYW5nZSBpbiB1bml0cyBzb2xkIGVxdWFsIHRvIHRoZSByZWdyZXNzaW9uIGNvZWZmaWNpZW50IGZvciBwcmljZS4gSWYgdGhlIGNvZWZmaWNpZW50IGlzIG5lZ2F0aXZlIGFuZCBzaWduaWZpY2FudCwgaXQgbWVhbnMgdGhhdCByYWlzaW5nIHRoZSBwcmljZSBieSBhIGRvbGxhciBsZWFkcyB0byBzZWxsaW5nIGZld2VyIHVuaXRzIG9uIGF2ZXJhZ2UuCgojIFF1ZXN0aW9uIDYKCkVzdGltYXRlIHRoZSBvd24tcHJpY2UgZWxhc3RpY2l0eSBmb3IgUHJvZHVjdDI2MDIuIEluIHlvdXIgY29uY2x1c2lvbiwgZ2l2ZSBhbiBleGFtcGxlIG9mIHdoYXQgIlByb2R1Y3QyNjAyIiBtaWdodCBhY3R1YWxseSBiZSAobWFueSBwb3NzaWJsZSBjb3JyZWN0IGFuc3dlcnMpIGFuZCBqdXN0aWZ5IHdoeSB5b3UgY2hvc2UgeW91ciBleGFtcGxlIHByb2R1Y3QuCgojIyBRNiBDb2RlCgpgYGB7cn0KIyBGaXQgYSBsb2ctbG9nIHJlZ3Jlc3Npb24gbW9kZWwgdG8gZXN0aW1hdGUgcHJpY2UgZWxhc3RpY2l0eQptb2RlbF9xNiA8LSBsbShsb2dfdW5pdHNfc29sZCB+IGxvZ19wcmljZSwgZGF0YSA9IHE1X2FuZF9xNl9kYXRhKQpzdW1tYXJ5KG1vZGVsX3E2KQoKIyBFeHRyYWN0IHRoZSBlbGFzdGljaXR5IGNvZWZmaWNpZW50CmVsYXN0aWNpdHkgPC0gY29lZihtb2RlbF9xNilbImxvZ19wcmljZSJdCmBgYAoKIyMgUTYgQ29uY2x1c2lvbgoKVGhlIGVzdGltYXRlZCBvd24tcHJpY2UgZWxhc3RpY2l0eSBzaG93cyB0aGF0IGEgMSUgaW5jcmVhc2UgaW4gcHJpY2UgbGVhZHMgdG8gYSBwcm9wb3J0aW9uYWwgZGVjcmVhc2UgaW4gdW5pdHMgc29sZC4gRm9yIGV4YW1wbGUsIFByb2R1Y3QyNjAyIGNvdWxkIGJlIHByaW50ZXIgcGFwZXIsIHNpbmNlIGRlbWFuZCBmb3Igc3VjaCBiYXNpYyBvZmZpY2Ugc3VwcGxpZXMgaXMgdHlwaWNhbGx5IG5vdCB2ZXJ5IHNlbnNpdGl2ZSB0byBwcmljZSBjaGFuZ2VzLgoKIyBRdWVzdGlvbiA3CgpFdmFsdWF0ZSB0aGUgcGVyZm9ybWFuY2Ugb2YgeW91ciByZWdyZXNzaW9ucyBpbiBRNSBhbmQgUTYgYnkgcGVyZm9ybWluZyBMZWF2ZS1vbmUtb3V0IENyb3NzLXZhbGlkYXRpb24gKExPT0NWKS4gRG9lcyBvbmUgbW9kZWwgcGVyZm9ybSBiZXR0ZXIgdGhhbiB0aGUgb3RoZXIgKmFuZCogaG93IGRvIHlvdSBrbm93IHRoYXQ/IElmIHRoZXJlIGlzIGEgZGlmZmVyZW5jZSBpbiBwZXJmb3JtYW5jZSwgd2hhdCBpcyB5b3VyIGV4cGxhbmF0aW9uIGZvciB3aHkgdGhhdCBpcyB0aGUgY2FzZT8KCkhpbnQ6IHdoaWxlIG5vdCBzdHJpY3RseSByZXF1aXJlZCwgeW91IG1heSB3YW50IHRvIHVzZSB0aGUga19jdigpIGZ1bmN0aW9uCgojIyBRNyBDb2RlCgpgYGB7cn0KIyBGb3IgUTUgTGluZWFyIFJlZ3Jlc3Npb24gTW9kZWwgKHVuaXRzX3NvbGQgfiBwcmljZSkKIyBQZXJmb3JtIExPT0NWIG1hbnVhbGx5Cm4gPC0gbnJvdyhxNV9hbmRfcTZfZGF0YSkKbXNlX3E1IDwtIG51bWVyaWMobikKCmZvciAoaSBpbiAxOm4pIHsKICB0cmFpbl9kYXRhIDwtIHE1X2FuZF9xNl9kYXRhWy1pLCBdCiAgdGVzdF9kYXRhIDwtIHE1X2FuZF9xNl9kYXRhW2ksIF0KICAKICBtb2RlbF9xNV9sb29jdiA8LSBsbSh1bml0c19zb2xkIH4gcHJpY2UsIGRhdGEgPSB0cmFpbl9kYXRhKQogIHByZWQgPC0gcHJlZGljdChtb2RlbF9xNV9sb29jdiwgbmV3ZGF0YSA9IHRlc3RfZGF0YSkKICBtc2VfcTVbaV0gPC0gKHRlc3RfZGF0YSR1bml0c19zb2xkIC0gcHJlZCleMgp9Cgpsb29jdl9tc2VfcTUgPC0gbWVhbihtc2VfcTUpCgojIEZvciBRNiBMb2ctTG9nIEVsYXN0aWNpdHkgTW9kZWwgKGxvZ191bml0c19zb2xkIH4gbG9nX3ByaWNlKQojIFBlcmZvcm0gTE9PQ1YgbWFudWFsbHkgKGVuc3VyZSBwcmVkaWN0aW9ucyBhcmUgZXhwb25lbnRpYXRlZCkKbXNlX3E2IDwtIG51bWVyaWMobikKCmZvciAoaSBpbiAxOm4pIHsKICB0cmFpbl9kYXRhIDwtIHE1X2FuZF9xNl9kYXRhWy1pLCBdCiAgdGVzdF9kYXRhIDwtIHE1X2FuZF9xNl9kYXRhW2ksIF0KICAKICBtb2RlbF9xNl9sb29jdiA8LSBsbShsb2dfdW5pdHNfc29sZCB+IGxvZ19wcmljZSwgZGF0YSA9IHRyYWluX2RhdGEpCiAgbG9nX3ByZWQgPC0gcHJlZGljdChtb2RlbF9xNl9sb29jdiwgbmV3ZGF0YSA9IHRlc3RfZGF0YSkKICBwcmVkIDwtIGV4cChsb2dfcHJlZCkgIyBDb252ZXJ0IGJhY2sgdG8gb3JpZ2luYWwgdW5pdHMKICBtc2VfcTZbaV0gPC0gKHRlc3RfZGF0YSR1bml0c19zb2xkIC0gcHJlZCleMgp9Cgpsb29jdl9tc2VfcTYgPC0gbWVhbihtc2VfcTYpCgojIENvbXBhcmUgcGVyZm9ybWFuY2UKY2F0KCJMT09DViBNU0UgZm9yIFE1IExpbmVhciBNb2RlbDoiLCBsb29jdl9tc2VfcTUsICJcbiIpCmNhdCgiTE9PQ1YgTVNFIGZvciBRNiBMb2ctTG9nIE1vZGVsOiIsIGxvb2N2X21zZV9xNiwgIlxuIikKYGBgCgojIyBRNyBDb25jbHVzaW9uCgpUaGUgbW9kZWwgd2l0aCB0aGUgbG93ZXIgTE9PQ1YgTVNFIGlzIGJldHRlci4gQSBsb3dlciBNU0UgZm9yIHRoZSBsb2ctbG9nIG1vZGVsIChRNikgc2hvd3MgaXQgYmV0dGVyIGNhcHR1cmVzIGVsYXN0aWNpdHkgKHR5cGljYWwgZm9yIHBlcmNlbnRhZ2UgY2hhbmdlcyBpbiBlY29ub21pY3MpLCB3aGlsZSBhIGJldHRlciBsaW5lYXIgbW9kZWwgKFE1KSBzdWdnZXN0cyBhIHN0cmFpZ2h0Zm9yd2FyZCBwcmljZS1xdWFudGl0eSByZWxhdGlvbnNoaXAuIFRoZSBjaG9pY2UgaGluZ2VzIG9uIHRoZSBkYXRh4oCZcyB1bmRlcmx5aW5nIHBhdHRlcm4gKGxpbmVhciB2cy4gbXVsdGlwbGljYXRpdmUpLgoKIyBRdWVzdGlvbiA4CgpZb3UgYXJlIGRvaW5nIG1hcmtldCByZXNlYXJjaCBmb3IgYSBjb21wYW55IHRoYXQgc2VsbHMgdGhyZWUgcHJvZHVjdHM6IEEsIEIsIGFuZCBDLiBUaGV5IGJlbGlldmUgdGhlcmUgaXMgYSByZWxhdGlvbnNoaXAgYmV0d2VlbiB0aGUgbnVtYmVyIG9mIHVuaXRzIG9mIEEgdGhleSBzZWxsIGFuZCB0aGUgcHJpY2VzIG9mIEIgYW5kIEMuIFlvdSBoYXZlIGRhdGEgb24gaG93IG1hbnkgdW5pdHMgb2YgUHJvZHVjdCBBIHdlcmUgc29sZCBhdCB2YXJpb3VzIHByaWNlcyBmb3IgUHJvZHVjdHMgQiBhbmQgQy4KCkNhbGN1bGF0ZSB0aGUgY3Jvc3MgcHJpY2UgZWxhc3RpY2l0aWVzIGZvciBBIHJlbGF0aXZlIHRvIEIgYW5kIEEgcmVsYXRpdmUgdG8gQy4gQmFzZWQgb24geW91ciByZXN1bHRzLCBnaXZlIGFuIGV4YW1wbGUgb2Ygd2hhdCBQcm9kdWN0IEEsIEIsIGFuZCBDIG1pZ2h0IGFjdHVhbGx5IGJlIChtYW55IHBvc3NpYmxlIGNvcnJlY3QgYW5zd2VycykuIEp1c3RpZnkgd2h5IHlvdSBjaG9zZSB5b3VyIGV4YW1wbGUgcHJvZHVjdHMuCgpIaW50OiBJdCBpcyBvbmx5IG5lY2Vzc2FyeSB0byBydW4gb25lIHJlZ3Jlc3Npb24KCiMjIFE4IENvZGUKCmBgYHtyfQojIEZpdCBhIGxvZy1sb2cgcmVncmVzc2lvbiB0byBjYWxjdWxhdGUgY3Jvc3MtcHJpY2UgZWxhc3RpY2l0aWVzCm1vZGVsX3E4IDwtIGxtKGxvZyh1bml0c19zb2xkX0EpIH4gbG9nKHByaWNlX0IpICsgbG9nKHByaWNlX0MpLCBkYXRhID0gcThfZGF0YSkKc3VtbWFyeShtb2RlbF9xOCkKCiMgRXh0cmFjdCBlbGFzdGljaXR5IGNvZWZmaWNpZW50cwplbGFzdGljaXR5X0IgPC0gY29lZihtb2RlbF9xOClbImxvZyhwcmljZV9CKSJdCmVsYXN0aWNpdHlfQyA8LSBjb2VmKG1vZGVsX3E4KVsibG9nKHByaWNlX0MpIl0KYGBgCiMjIFE4IENvbmNsdXNpb24KClRoZSByZXN1bHRzIHNob3cgdGhhdCBQcm9kdWN0IEEgYW5kIEIgYXJlIHN1YnN0aXR1dGVzIChwb3NpdGl2ZSBlbGFzdGljaXR5KSwgd2hpbGUgQSBhbmQgQyBhcmUgY29tcGxlbWVudHMgKG5lZ2F0aXZlIGVsYXN0aWNpdHkpLiBGb3IgZXhhbXBsZSwgaWYgUHJvZHVjdCBBIGlzIGNvZmZlZSwgUHJvZHVjdCBCIGNvdWxkIGJlIHRlYSAoYSBzdWJzdGl0dXRlKSwgYW5kIFByb2R1Y3QgQyBjb3VsZCBiZSBzdWdhciAoYSBjb21wbGVtZW50KS4gQ2hhbmdlcyBpbiBwcmljZXMgb2YgQiBhbmQgQyBhZmZlY3QgdGhlIGRlbWFuZCBmb3IgQSBhY2NvcmRpbmdseS4KCiMtLS0tLS0tLS0tLS0tLS0tLS0tLQojIG1vZGVsX3E4IDwtIGxtKGxvZyh1bml0c19zb2xkX0EpIH4gbG9nKHByaWNlX0IpICsgbG9nKHByaWNlX0MpLCBkYXRhID0gcThfZGF0YSkKc3VtbWFyeShtb2RlbF9xOCkKIy0tLS0tLS0tLS0tLS0tLS0tLS0tCgojIFF1ZXN0aW9uIDkKCllvdSB3b3JrIGZvciBhIGNvbXBhbnkgdGhhdCBsYXVuY2hlcyBkaWZmZXJlbnQgYWQgY2FtcGFpZ25zIG9uIGEgcmVndWxhciBiYXNpcy4gWW91ciB0YXNrIGlzIHRvIGRldGVybWluZSB3aGF0IGZhY3RvcnMgaW1wYWN0IHRoZSBsaWtlbGlob29kIHRoYXQgdGhlIGFkIGdldHMgY2xpY2tlZCBvbi4gU2ltdWxhdGUgdGhlIGZvbGxvd2luZyBkYXRhIGFuZCB0YWtlIGZhbWlsaWFyaXplIHlvdXJzZWxmIHdpdGggdGhlIHZhcmlhYmxlcyBpbiBxOV9hbmRfcTEwX2RhdGEuCgpSdW4gYSBsb2dpc3RpYyByZWdyZXNzaW9uIHdpdGggYWxsIHZhcmlhYmxlcyBpbiBxOV9hbmRfcTEwX2RhdGEuIEludGVycHJldCB0aGUgQU1FIChhdmVyYWdlIG1hcmdpbmFsIGVmZmVjdCkgY29sdW1uIGZvciBldmVyeSBwcmVkaWN0b3IgaW4gdGhlIHJlZ3Jlc3Npb24uCgpOb3RlOiB5b3Ugc2hvdWxkIHVzZSBzbWFydF9sb2dpdCgpCgojIyBROSBDb2RlCgpgYGB7cn0KIyBDaGVjayB0aGUgYWN0dWFsIGNvbHVtbiBuYW1lcyBpbiB5b3VyIGRhdGEKY29sbmFtZXMocTlfYW5kX3ExMF9kYXRhKQoKIyBTdXBwb3NlIHlvdXIgY29sdW1ucyBhcmU6ICJjbGlja2VkIiwgImFkX2xlbmd0aCIsICJhZF90eXBlIiwgInRhcmdldF9hdWRpZW5jZSIsICJidWRnZXQiCiMgSWYgdGhlIG5hbWVzIGFyZSBkaWZmZXJlbnQsIHVzZSB0aGUgZXhhY3QgbmFtZXMgeW91IHNlZSBmcm9tIGNvbG5hbWVzKCkKCiMgUnVuIHRoZSBsb2dpc3RpYyByZWdyZXNzaW9uIHVzaW5nIHNtYXJ0X2xvZ2l0IChpZiBhdmFpbGFibGUgaW4geW91ciBlbnZpcm9ubWVudCkKc21hcnRfbG9naXQoY2xpY2sgfiAuLCBkYXRhID0gcTlfYW5kX3ExMF9kYXRhKQpgYGAKIyMgUTkgQ29uY2x1c2lvbgoKVGhlIEFNRSBzaG93cyBob3cgZWFjaCBwcmVkaWN0b3IgY2hhbmdlcyB0aGUgcHJvYmFiaWxpdHkgb2YgYW4gYWQgYmVpbmcgY2xpY2tlZC4gUG9zaXRpdmUgdmFsdWVzIGluY3JlYXNlIGNsaWNrIGxpa2VsaWhvb2QsIG5lZ2F0aXZlIHZhbHVlcyBkZWNyZWFzZSBpdCwgYW5kIGxhcmdlciB2YWx1ZXMgbWVhbiBhIGJpZ2dlciBpbXBhY3QuIFVzZSB0aGVzZSByZXN1bHRzIHRvIGlkZW50aWZ5IHdoaWNoIGZhY3RvcnMgbW9zdCBpbmZsdWVuY2UgYWQgY2xpY2tzLgoKIyBRdWVzdGlvbiAxMAoKUnVuIGEgTGluZWFyIFByb2JhYmlsaXR5IE1vZGVsIChMUE0sIGFrYSBPTFMpIHdpdGggYWxsIHZhcmlhYmxlcyBpbiBxOV9hbmRfcTEwX2RhdGEuIENvbXBhcmUgdGhlIG1vZGVsIGZpdCBvZiB0aGUgTFBNIHRvIHRoZSBmaXQgb2YgdGhlIGxvZ2lzdGljIHJlZ3Jlc3Npb24gZnJvbSBROS4gRG9lcyBvbmUgbW9kZWwgZml0IHRoZSBkYXRhIGJldHRlciB0aGFuIHRoZSBvdGhlcj8gSWYgc28sIHdoeSBpcyB0aGF0IHRoZSBjYXNlPwoKIyMgUTEwIENvZGUKCmBgYHtyfQojIExpbmVhciBQcm9iYWJpbGl0eSBNb2RlbCAoTFBNKQptb2RlbF9scG0gPC0gbG0oY2xpY2sgfiAuLCBkYXRhID0gcTlfYW5kX3ExMF9kYXRhKQoKIyBMb2dpc3RpYyBSZWdyZXNzaW9uCm1vZGVsX2xvZ2l0IDwtIGdsbShjbGljayB+IC4sIGRhdGEgPSBxOV9hbmRfcTEwX2RhdGEsIGZhbWlseSA9IGJpbm9taWFsKQoKIyBDb21wYXJlIG1vZGVsIGZpdCB1c2luZyBBSUMKY2F0KCJMUE0gQUlDOiIsIEFJQyhtb2RlbF9scG0pLCAiXG4iKQpjYXQoIkxvZ2lzdGljIFJlZ3Jlc3Npb24gQUlDOiIsIEFJQyhtb2RlbF9sb2dpdCksICJcbiIpCmBgYAoKIyMgUTEwIENvbmNsdXNpb24KClRoZSBsb2dpc3RpYyByZWdyZXNzaW9uIGZpdHMgYmV0dGVyIHRoYW4gdGhlIExQTSBiZWNhdXNlIGl0IGtlZXBzIHByZWRpY3RlZCBwcm9iYWJpbGl0aWVzIGJldHdlZW4gMCBhbmQgMSBhbmQgaXMgbW9yZSBhcHByb3ByaWF0ZSBmb3IgYmluYXJ5IG91dGNvbWVzLgoK