MLR section 25 Upper Confidence Bound

Importing the dataset

dataset <-  read.csv('G:\\RStudio\\udemy\\ml\\Machine Learning AZ\\Part 6 - Reinforcement Learning\\Section 32 - Upper Confidence Bound (UCB)\\UCB\\Ads_CTR_Optimisation.csv')
head(dataset)

Implementing UCB algorithm

There is package to use UCB. So we will create the algorithm from scatch

# initialize variables we will be using
N <-  10000
d <-  10
ads_selected <-  integer(0)
numbers_of_selections <-  integer(d)
sums_of_rewards <-  integer(d)
total_reward <-  0
for (n in 1:N) {
  
  ad <-  0
  max_upper_bound <-  0
    for ( i in 1 : d) { 
      # dont use this if we are just starting out
      if (numbers_of_selections[i] > 0){
    average_reward <-  sums_of_rewards[i] / numbers_of_selections[i]
    delta_i <-  sqrt(3/2 * log(n) / numbers_of_selections[i])
    upper_bound <-  average_reward + delta_i
      } else {
        upper_bound = 1e400
      }
    if (upper_bound > max_upper_bound) {
      max_upper_bound <-  upper_bound
      ad <-  i
    }
  }
  ads_selected <-  append(ads_selected, ad)
  numbers_of_selections[ad] <- numbers_of_selections[ad] + 1
  reward <-  dataset[n,ad]
  sums_of_rewards[ad] <-  sums_of_rewards[ad] + reward
  total_reward <-  total_reward + reward
}

Visualising the resuls by using a histogram

hist(ads_selected, 
     col = "Blue",
     main = "Histogram of Ad Selections",
     xlab = "Ads",
     ylab = "Number of times each ad was selected")