Apply 3 Edited & Apply 4
Anil Eser
2024-09-20
Goal: Split our data and build a model Click [here for the data] https://github.com/rfordatascience/tidytuesday/tree/master/data/2022/2022-11-01
Import Data
horror_movies <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2022/2022-11-01/horror_movies.csv')
skimr:: skim(horror_movies)| Name | horror_movies |
| Number of rows | 32540 |
| Number of columns | 20 |
| _______________________ | |
| Column type frequency: | |
| character | 10 |
| Date | 1 |
| logical | 1 |
| numeric | 8 |
| ________________________ | |
| Group variables | None |
Variable type: character
| skim_variable | n_missing | complete_rate | min | max | empty | n_unique | whitespace |
|---|---|---|---|---|---|---|---|
| original_title | 0 | 1.00 | 1 | 191 | 0 | 30296 | 0 |
| title | 0 | 1.00 | 1 | 191 | 0 | 29563 | 0 |
| original_language | 0 | 1.00 | 2 | 2 | 0 | 97 | 0 |
| overview | 1286 | 0.96 | 1 | 1000 | 0 | 31020 | 0 |
| tagline | 19835 | 0.39 | 1 | 237 | 0 | 12513 | 0 |
| poster_path | 4474 | 0.86 | 30 | 32 | 0 | 28048 | 0 |
| status | 0 | 1.00 | 7 | 15 | 0 | 4 | 0 |
| backdrop_path | 18995 | 0.42 | 29 | 32 | 0 | 13536 | 0 |
| genre_names | 0 | 1.00 | 6 | 144 | 0 | 772 | 0 |
| collection_name | 30234 | 0.07 | 4 | 56 | 0 | 815 | 0 |
Variable type: Date
| skim_variable | n_missing | complete_rate | min | max | median | n_unique |
|---|---|---|---|---|---|---|
| release_date | 0 | 1 | 1950-01-01 | 2022-12-31 | 2012-12-09 | 10999 |
Variable type: logical
| skim_variable | n_missing | complete_rate | mean | count |
|---|---|---|---|---|
| adult | 0 | 1 | 0 | FAL: 32540 |
Variable type: numeric
| skim_variable | n_missing | complete_rate | mean | sd | p0 | p25 | p50 | p75 | p100 | hist |
|---|---|---|---|---|---|---|---|---|---|---|
| id | 0 | 1.00 | 445910.83 | 305744.67 | 17 | 146494.8 | 426521.00 | 707534.00 | 1033095.00 | ▇▆▆▅▅ |
| popularity | 0 | 1.00 | 4.01 | 37.51 | 0 | 0.6 | 0.84 | 2.24 | 5088.58 | ▇▁▁▁▁ |
| vote_count | 0 | 1.00 | 62.69 | 420.89 | 0 | 0.0 | 2.00 | 11.00 | 16900.00 | ▇▁▁▁▁ |
| vote_average | 0 | 1.00 | 3.34 | 2.88 | 0 | 0.0 | 4.00 | 5.70 | 10.00 | ▇▂▆▃▁ |
| budget | 0 | 1.00 | 543126.59 | 4542667.81 | 0 | 0.0 | 0.00 | 0.00 | 200000000.00 | ▇▁▁▁▁ |
| revenue | 0 | 1.00 | 1349746.73 | 14430479.15 | 0 | 0.0 | 0.00 | 0.00 | 701842551.00 | ▇▁▁▁▁ |
| runtime | 0 | 1.00 | 62.14 | 41.00 | 0 | 14.0 | 80.00 | 91.00 | 683.00 | ▇▁▁▁▁ |
| collection | 30234 | 0.07 | 481534.88 | 324498.16 | 656 | 155421.0 | 471259.00 | 759067.25 | 1033032.00 | ▇▅▅▅▅ |
Clean Data
data <- horror_movies %>%
# Log transform vote_average
mutate(vote_average = log1p(vote_average)) %>% # for zeroes: log1p(x) is the same as log(x+1)
# Treat multiple categories in genre_names
separate_rows(genre_names, sep = ", ") %>%
filter(status == "Released") %>%
select(id, vote_average, genre_names, overview, runtime) %>%
na.omit()Explore Data
data %>%
ggplot(aes(runtime, vote_average)) +
geom_point()
data %>%
group_by(runtime, vote_average) %>%
summarise(mean_group = mean(vote_average)) -> data2
data2 %>%
ggplot(aes(x= runtime, y= mean_group,
color= runtime, shape= vote_average,
group = runtime,
label = round(mean_group, 2))) +
scale_shape_binned() +
geom_point()
Prepare Data
data_binarized_tbl <- data %>%
select(-overview, -genre_names) %>%
binarize()
data_binarized_tbl %>% glimpse()## Rows: 62,252
## Columns: 11
## $ `id__-Inf_105927` <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ id__105927_387814 <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ id__387814_654747 <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ id__654747_Inf <dbl> 1, 1, 1, 1, 1, 1, 1, 1…
## $ `vote_average__-Inf_1.70474809223843` <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ vote_average__1.70474809223843_1.93152141160321 <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ vote_average__1.93152141160321_Inf <dbl> 1, 1, 1, 1, 1, 1, 1, 1…
## $ `runtime__-Inf_24` <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ runtime__24_84 <dbl> 0, 0, 0, 0, 0, 0, 0, 0…
## $ runtime__84_93 <dbl> 0, 0, 1, 1, 1, 0, 0, 0…
## $ runtime__93_Inf <dbl> 1, 1, 0, 0, 0, 1, 1, 1…Correlate
data_corr_tbl <- data_binarized_tbl %>%
correlate( `vote_average__-Inf_1.70474809223843` )
data_corr_tbl## # A tibble: 11 × 3
## feature bin correlation
## <fct> <chr> <dbl>
## 1 vote_average -Inf_1.70474809223843 1
## 2 vote_average 1.70474809223843_1.93152141160321 -0.585
## 3 vote_average 1.93152141160321_Inf -0.579
## 4 id 654747_Inf 0.237
## 5 id -Inf_105927 -0.233
## 6 runtime -Inf_24 0.188
## 7 runtime 93_Inf -0.178
## 8 runtime 84_93 -0.0929
## 9 runtime 24_84 0.0778
## 10 id 105927_387814 -0.0419
## 11 id 387814_654747 0.0386Plot
data_corr_tbl %>%
plot_correlation_funnel()
Build Models
Split Data
data <- sample_n(data, 100)
# Split into train and test data set
set.seed(1234)
data_split <- rsample::initial_split(data)
data_train <- training(data_split)
data_test <- testing(data_split)
# Further split training data set for cross-validation
set.seed(2345)
data_cv <- rsample::vfold_cv(data_train)
data_cv## # 10-fold cross-validation
## # A tibble: 10 × 2
## splits id
## <list> <chr>
## 1 <split [67/8]> Fold01
## 2 <split [67/8]> Fold02
## 3 <split [67/8]> Fold03
## 4 <split [67/8]> Fold04
## 5 <split [67/8]> Fold05
## 6 <split [68/7]> Fold06
## 7 <split [68/7]> Fold07
## 8 <split [68/7]> Fold08
## 9 <split [68/7]> Fold09
## 10 <split [68/7]> Fold10library(usemodels)## Warning: package 'usemodels' was built under R version 4.4.1usemodels::use_xgboost(vote_average ~., data = data_train)## xgboost_recipe <-
## recipe(formula = vote_average ~ ., data = data_train) %>%
## step_zv(all_predictors())
##
## xgboost_spec <-
## boost_tree(trees = tune(), min_n = tune(), tree_depth = tune(), learn_rate = tune(),
## loss_reduction = tune(), sample_size = tune()) %>%
## set_mode("classification") %>%
## set_engine("xgboost")
##
## xgboost_workflow <-
## workflow() %>%
## add_recipe(xgboost_recipe) %>%
## add_model(xgboost_spec)
##
## set.seed(6804)
## xgboost_tune <-
## tune_grid(xgboost_workflow, resamples = stop("add your rsample object"), grid = stop("add number of candidate points"))xgboost_recipe <-
recipe(formula = vote_average ~ ., data = data_train) %>%
step_other(genre_names, threshold = 0.05) %>%
step_dummy(all_nominal_predictors(), one_hot = TRUE) %>%
step_YeoJohnson(all_numeric_predictors())
xgboost_spec <-
boost_tree(
trees = tune(),
min_n = tune(),
tree_depth = tune(),
learn_rate = tune(),
loss_reduction = tune(),
sample_size = tune()
) %>%
set_mode("regression") %>%
set_engine("xgboost")
xgboost_workflow <-
workflow() %>%
add_recipe(xgboost_recipe) %>%
add_model(xgboost_spec)
# Tune the model
set.seed(6804)
xgboost_tune <- tune_grid(
xgboost_workflow,
resamples = data_cv,
grid = 5
)
xgboost_recipe %>% prep() %>% juice() %>% glimpse()## Rows: 75
## Columns: 82
## $ id <dbl> …
## $ runtime <dbl> …
## $ vote_average <dbl> …
## $ genre_names_Comedy <dbl> …
## $ genre_names_Horror <dbl> …
## $ genre_names_Thriller <dbl> …
## $ genre_names_other <dbl> …
## $ overview_A.big.game.hunter.brings.a.killer.leopard.to.his.private.island.and.turns.it.loose.so.he.can.hunt.it.down..However..unexpected.visitors.arrive.at.the.island.and.interrupt.his.hunt..Meanwhile..the.leopard.begins.to.hunt.the.inhabitants.of.the.island. <dbl> …
## $ overview_A.bunch.of.really.cool.stuff.happens..just.watch.it..Trust.me. <dbl> …
## $ overview_A.couple.of.slackers..apartment.is.invaded.by.a.garden.shear.wielding.maniac..think.The.Burning..bent.on.brutally.murdering.everyone.in.his.path. <dbl> …
## $ overview_A.cowardly.priest.must.exorcise.a.demon.from.the.body.of.a.teenager.whose.life.seems.to.have.changed.for.the.better. <dbl> …
## $ overview_A.driven.young.medical.resident.who.transfers.to.a.rural.hospital.for.a.fresh.start..There..the.demons.of.his.past.start.to.catch.up.to.him.when.he.becomes.consumed.by.the.case.of.a.young.asthma.patient. <dbl> …
## $ overview_A.family.on.vacation.stops.at.a.roadside.restaurant.where.a.gang.of.criminals.are.holding.the.employees.hostage..When.one.of.the.hostages.tries.to.enlist.their.help..the.family.is.added.to.the.group..Disagreements.on.how.the.hostages.should.be.treated.leads.to.dissent.among.the.criminals..who.are.awaiting.the.arrival.of.their.leader...The.leader..for.unexplained.reasons..has.all.the.stolen.money...Some.hostages.are.killed..when.the.leader.arrives..the.hostages.overpower.them.and.free.themselves. <dbl> …
## $ overview_A.gang.of.ruthless.highway.killers.kidnap.a.wealthy.couple.traveling.cross.country.only.to.shockingly.discover.that.things.are.not.what.they.seem. <dbl> …
## $ overview_A.group.of.college.kids.go.after.hours.bowling..The.fun.is.suddenly.halted.when.abnormal.things.begin.happening.on.the.13th.alley..Everyone.runs.to.escape..but.all.exits.have.been.boarded.up..Who.is.out.for.revenge. <dbl> …
## $ overview_A.group.of.friends.sneak.into.an.old.abandoned.house.on.Halloween.night.haunted.by.hungry.ghosts. <dbl> …
## $ overview_A.group.of.people.fight.for.their.small.town.after.vampires.take.over.the.world. <dbl> …
## $ overview_A.group.of.scientists.have.developed.the.Resonator..a.machine.which.allows.whoever.is.within.range.to.see.beyond.normal.perceptible.reality..But.when.the.experiment.succeeds..they.are.immediately.attacked.by.terrible.life.forms. <dbl> …
## $ overview_A.guy.comes.home.from.college..Then.the.end.begins. <dbl> …
## $ overview_A.jealous.and.overprotective.friend.who.wants.his.Queen.to.himself.clears.out.anyone.who.takes.up.her.time. <dbl> …
## $ overview_A.litte.horror.story.inspired.by.real.events.from.a.certain.shared.flat.I.used.to.live.in <dbl> …
## $ overview_A.little.girl.is.afraid.of.the.wolf.living.with.her.in.the.house. <dbl> …
## $ overview_A.lovelorn.underdog.buys.a.novelty.meteorite.as.part.of.a.romantic.gesture.to.win.his.ex.back..but.when.the.gift.proves.to.have.a.mind.of.its.own..he.must.purge.himself.of.the.parasitic.entity.before.it.consumes.him. <dbl> …
## $ overview_A.married.man.on.a.business.trip.checks.into.a.hotel..The.hotel.manager.s.daughter.falls.for.him.at.first.sight..Rejected.by.the.man..she.embarks.on.a.journey.of.revenge... <dbl> …
## $ overview_A.masked.man.is.killing.a.group.of.underworld.hoods..which.has.the.police.and.the.hoods.equally.baffled. <dbl> …
## $ overview_A.motorcycle.gang.kidnaps.a.young.woman..Josie..from.a.diner.and.brutally.kills.her..Many.years.later..the.girl.s.father.finds.a.magic.crystal.that.can.bring.the.life.back.to.dead.objects..He.uses.it.to.re.animate.his.daughter..He.lets.her.seduce.any.young.man.that.comes.to.visit.the.small.town.and.then.kills.them..Four.young.students..two.boys.and.two.girls.spend.a.vacation.near.the.town..Hellgate..They.hear.about.the.story.and.get.involved. <dbl> …
## $ overview_A.pitch.black.comedy.about.Vivian..a.lonely.serial.killer.whose.three.loves.in.life.are.math..accounting..and.killing.random.people..This.all.changes.when.she.meets.Alex..the.man.of.her.dreams. <dbl> …
## $ overview_A.psychic.housewife.and.her.husband.accidentally.find.a.kidnapped.girl..But.instead.of.informing.the.police..they.hatch.a.scheme.to.get.famous.by.working.with.the.police.as.a.psychic.consultant.to..find..the.girl..And.then..things.start.to.go.terribly.wrong. <dbl> …
## $ overview_A.schoolgirl.was.murdered.seven.years.ago..and.the.case.was.never.solved..now..the.murderer.seems.to.be.back. <dbl> …
## $ overview_A.short.collection.of.local.legends.and.ghost.stories.about.Erie..Pennsylvania..and.its.surrounding.areas..Produced.by.and.aired.on.WQLN.Channel.54.Erie. <dbl> …
## $ overview_A.speed.boat.racer.suffers.amnesia.after.an.boating.accident.and.goes.to.his.isolated.villa.to.recuperate..His.relationship.with.his.wife.is.troubled..so.he.starts.an.affair..sort.of..with.the.another.woman..which.results.in.one.of.the.three.of.them.being.shot..But.there.are.several.more.twists..and.more.murders..to.come. <dbl> …
## $ overview_A.Taoist.priest.protests.against.the.idea.of.a.Buddhist.priest.being.his.neighbour..They.often.argue.with.each.other.until.a.vampire.is.at.large.and.threatens.the.villagers..safety. <dbl> …
## $ overview_A.woman.acquires.a.box.with.a.seal.made.of.white.jade..it.is.rumored.that.the.box.belonged.to.the.first.Chinese.emperor...She.journeys.to.Hong.Kong.in.an.attempt.to.sell.the.artifact. <dbl> …
## $ overview_A.young.drug.addict.who.has.lost.touch.with.her.rich.family.hits.rock.bottom.when.she.sells.her.new.born.baby.to.child.traffickers..She.soon.regrets.her.decision.and.finds.the.courage.and.strength.to.get.him.back..To.do.so..she.must.follow.a.trail.into.the.woods.and.find.an.eerie.house.where.the.old.lady.she.met..her.albino.servant.and.a.girl.with.a.strange.gait..all.live.together..Nothing.could.have.prepared.her.for.the.horror.she.is.about.to.discover. <dbl> …
## $ overview_A.young.writer.traumatized.by.the.death.of.his.wife.and.kid..describes.his.disturbing.experiences.to.a.psychiatrist..A.mysterious.man..hidden.under.an.umbrella..forces.him.to.finish.his.book.regardless.the.cost.of.its.consequences. <dbl> …
## $ overview_After.receiving.word.about.a.mysterious.carcass.skeleton.unearthed.in.the.Arizona.desert..a.father.and.his.daughter.decide.to.remove.it.from.the.burial.grounds.for.further.study..Once.they.do.so..they..as.well.as.the.town..are.besieged.by.a.colony.of.gargoyles.living.in.some.nearby.caverns. <dbl> …
## $ overview_After.several.years.without.writing.anything.that.could.aid.him.with.his.financial.troubles..Gabriel.Balaguer.spends.his.nights.in.despair.under.the.pressure.of.creating.best.selling.success..something.as.good.as.his.first.work..To.accomplish.freshness.and.originality..he.will.strife.further.than.we.are.able.to.see. <dbl> …
## $ overview_After.the.tragic.loss.of.his.wife.battling.the.forces.of.darkness..Gabriel.is.persuaded.to.rejoin.his.former.team.of.demon.hunters.traveling.from.relatively.obscurity.in.America.to.the.deep.unknown.regions.of.Europe..He.is.joined.by.his.estranged.best.friend.Michael.who.harbors.a.dark.secret..the.mystical.Hannah.whose.visions.predict.the.future..and.a.motley.crew.of.demon.slayers..After.losing.part.of.his.team.Gabriel.must.confront.his.tragic.past.and.decide.who.really.is.friend.or.foe. <dbl> …
## $ overview_An.ad.executive.uses.psychic.powers.to.manipulate.women.and.competitors. <dbl> …
## $ overview_An.ordinary.day.in.a.cafe.takes.a.bizarre.twist.for.Alice.and.Sharon. <dbl> …
## $ overview_Awoken.by.screams.in.the.night..young.Evelyn.Whipley.is.found.drenched.in.blood.and.mysteriously.bound.to.the.bed...A.Gothic.Horror.set.in.the.1940s..a.psychosexual.thriller.disguised.as.traditional.Ghost.Story. <dbl> …
## $ overview_Based.on.a.traditional.Japanese.ghost.story..the.movie.follows.a.man.who.falls.deeply.in.love.with.a.mysterious.woman..only.to.find.she.belongs.to.the.world.of.the.dead. <dbl> …
## $ overview_Born.in.the.New.Jersey.public.housing..the.young.Johnny.Jones..Robert.Crayton..in.his.life.has.known.only.abuse.and.poverty..Pushed.over.the.edge..Johnny.Jones.suffers.a.mental.breakdown.and.ignites.a.killing.spree.never.seen.in.history..leaving.a.trail.of.death.and.carnage.everywhere.he.goes..While.the.tensions.and.the.murders.only.get.worse..the.city.calls.Detective.Cutchin..Tom.Sizemore..to.solve.the.case.and.bring.Johnny.Jones.to.justice. <dbl> …
## $ overview_College.students.exploring.the.Ozark.Mountains.for.a.school.assignment..stumble.upon.a.group.of.scary..redneck.cave.dwellers. <dbl> …
## $ overview_Death.in.all.it.s.faces.and.stages..From.the.horrors.of.Buchenwald.to.the.devastation.of.Hiroshima..From.the.political.assassinations.of.the.second.half.of.the.20th.century.to.the.bloody.feeding.frenzy.of.the.pythons.of.Burma..Burned.on.to.the.screen.like.napalm.victims.of.Vietnam..Followed.by..Death.in.Focus..part.2. <dbl> …
## $ overview_Dr..Frankensteins..daughter..who.is.in.love.with.the.aging.lab.assistant.Marshall..continues.with.her.fathers.experiments.and.attempts.to.transplant.Marshall.s.brain.into.a.new.body.to.prolong.his.life. <dbl> …
## $ overview_Eight.medical.students.on.a.ski.trip.to.Norway.discover.that.Hitler.s.horrors.live.on.when.they.come.face.to.face.with.a.battalion.of.zombie.Nazi.soldiers.intent.on.devouring.anyone.unfortunate.enough.to.wander.into.the.remote.mountains.where.they.were.once.sent.to.die. <dbl> …
## $ overview_Five.girls.stray.from.society.on.the.hunt.for.something.more.special...you.can.find.them.if.you.follow.the.sound. <dbl> …
## $ overview_Horror...When.a.group.of.old.friends.decide.to.camp.out.in.the.same.woods.where.one.of.their.pals.was.killed.four.years.earlier..their.weekend.reunion.turns.grisly.once.more.as.a.mysterious.creature.begins.to.haunt.them....and.hunt.them.down..At.the.advice.of.a.geneticist..Chase.Masterson...the.campers.attempt.to.leave.the.forest..but.it.may.be.too.late..the.murderous.being.seems.hell.bent.on.destruction..no.matter.what.the.cost.....Kristin.Herold..Chase.Masterson..Chris.Hoffman <dbl> …
## $ overview_Host.Mr.Lobo.investigates.a.real.haunted.house.with.his.own.hilarious.group.of..Paranoid.Investigators..comprised.entirely.of.fellow.TV.horror.hosts..Many.fun.clips.and.horror.trailers.are.presented. <dbl> …
## $ overview_Illustrator.Natalie.is.forced.to.create.four.terrifying.stories.for.the.new.comic.book..Hi.Fear.. <dbl> …
## $ overview_In.1997..four.campers.went.missing.during.the.Phoenix.Lights.incident.and.their.bodies.were.never.found..The.Phoenix.Tapes.97.is.comprised.of.their.final.footage..This.film.is.dedicated.to.the.lives.lost.during.the.events.displayed..To.their.families.and.friends.and.to.the.authorities.who.continue.to.search.for.answers....The.tapes.were.found.by.Dustin.Miller.s.father.two.years.after.Dustin.was.mysteriously.shot.and.killed.at.a.traffic.stop.in.Texas..Peter.Miller.believes.his.son.was.acting.as.a.whistleblower.and.had.smuggled.the.tapes.from.his.classified.government.agency.in.an.effort.to.expose.the.truth.as.to.what.really.happened.to.the.four.missing.campers.seen.in.the.tapes....After.searching.every.newspaper.and.media.publication.around.Phoenix.during.that.time.period..we.only.found.one.initial.announcement.that.four.campers.were.reported.missing..as.well.as.a.formal.police.report.folder.with.local.PD.but.the.folder.had.no.files.inside. <dbl> …
## $ overview_In.a.desolate.parking.garage..a.woman.grows.wary.when.a.second.device.connects.to.her.car.s.bluetooth. <dbl> …
## $ overview_In.a.world.torn.by.a.war.between.vampires.and.angels..a.female.vigilante.on.the.hunt.for.her.missing.sisters.is.lured.into.the.den.of.a.vampire. <dbl> …
## $ overview_In.the.late.90s..a.video.archivist.unearths.a.series.of.sinister.pirate.broadcasts.and.becomes.obsessed.with.uncovering.the.dark.conspiracy.behind.them. <dbl> …
## $ overview_In.wild.18th.century.Britain.a.lonely.woman.discovers.that.her.menstrual.cycle.is.of.interest.to.a.stranger. <dbl> …
## $ overview_It.s.Halloween.night.and.Renee.is.madly.in.love.with.Jim..On.their.way.to.a.party..Jim.detours.to.a.haunted.house.where.Renee.is.forced.to.confront.a.terrifying.situation. <dbl> …
## $ overview_Italian.horror.movie.written.and.directed.by.Pupi.Avati. <dbl> …
## $ overview_Izzy.takes.a.babysitting.job.that.appears.all.too.easy.at.first..But.between.multiple.baby.dolls.around.the.house..as.well.as.strange.noises..Izzy.soon.regrets.taking.the.job. <dbl> …
## $ overview_Jack.commits.murder.because.Bob..the.man.on.the.television..told.him.to <dbl> …
## $ overview_Mild.mannered.bookstore.owner.and.comic.book.dealer.Flecks.Crushe.leads.a.double.life.as.a.pathological.mob.enforcer.for.his.ruthless.boss..Senator.Corman..When.Flecks.is.seen.acting.strangely.his.cohorts.question.his.sanity..After.Flecks.kills.a.member.of.his.crew.for.working.with.his.arch.nemesis.Romero..Mike...Lenny.decide.it.s.high.time.to.fit.Flecks.with.a.pine.overcoat..As.the.dark.and.ludicrous.world.of.Face.Eater.unfolds..the.mobsters.fight.it.out.the.only.way.they.can..And.you.will.be.on.the.edge.of.your.seat.trying.to.decide.who.will.avoid.having.their.face.eaten. <dbl> …
## $ overview_Moneyless..ocean.exploring.gigolo.and.his.worldly.lady.friend.are.forced.to.take.part.in.looking.into.the.possibility.that.aggression...utilizing.marine.life...is.being.planned. <dbl> …
## $ overview_Nam.is.not.your.typical.teenager..Sure..he.spends.a.little.too.much.time.on.his.computer..and.tries.a.little.too.hard.to.meet.the.gorgeous.stewardess.who.lives.in.his.building..But.recently.three.of.Nam.s.friends.have.committed.suicide..which.isn.t.all.bad.since..as.the.title.implies..he.sees.ghosts..However..these.suicides.only.foreshadow.weirder.things.to.come..Soon.his.brother.Tung..along.with.Chee..his.stewardess.crush..begin.to.act.strangely..while.his.visions.of.ghostly.children.and.other.phantoms.grow.more.frequent..With.aid.from.his.brother.s.girlfriend.Charlie..Nam.sets.out.to.investigate.his.haunted.building.s.past.and.help.these.malevolent.spirits.cross.over. <dbl> …
## $ overview_Old.school.friends.go.on.a.paintball.trip.miles.away.from.civilization..But.things.go.horribly.wrong.when.they.discover.a.cold.blooded.masked.killer.is.amongst.them..Now..they.must.fight.to.survive.their.school.reunion. <dbl> …
## $ overview_On.April.5th..2011.filmmakers.Jack.Hamill.and.Brenda.Hill.were.hired.to.make.a.documentary.about.serial.killer.Edward.Payne..3.weeks.later.Jack.Hamil.s.partially.decomposed.remains.were.found.in.an.abandoned.field.outside.a.small.town.in.Oregon..An.investigation.into.the.crime.led.police.to.a.farmhouse.on.the.edge.of.town..What.they.found.there.would.shake.the.tight.knit.community.to.the.core..Nearly.a.dozen.young.women.were.found.decomposing.in.the.basement..Stacks.of.documentary.tapes.were.left.on.the.kitchen.table..And.the.hunt.for.one.of.America.s.most.notorious.serial.killers.was.set.to.begin..Witness.the.birth.of.a.monster.with.this.behind.the.scenes.look.into.the.mind.of.a.killer. <dbl> …
## $ overview_Our.story.begins.when.a.group.of.girls.from.a..halfway..program.are.hired.to.organize.and.document.the.contents.of.a.huge.old.building.for.it.s.new.owners...What.they.find.inside.is.a.terrifying.trip.into.the.mouth.of.Hell.itself...The.old.building.houses.dark.secrets.that.result.in.brutal.acts.of.murder..but.the.ones.murdered.are.the.lucky.ones..it.s.the.ones.who.live.that.have.it.face.the.true.Horror.and.the.shocking.truth.that.lies.at.the.heart.of.the.evil.within.the.building. <dbl> …
## $ overview_Penny.finds.herself.getting.closer.to.office.colleague.Gregg..but.her.dark.secret.might.get.in.the.way.of.romance. <dbl> …
## $ overview_The.director.of.a.school.is.surprised.by.a.student.when.murdering.someone..He.must.kill.the.only.eyewitness.too..but.his.only.clue.is.a.number.32.shoeprint. <dbl> …
## $ overview_The.innocent.children.in.the.village.are.happy.to.play.in.the.village..singing.nursery.rhymes..Such.a.scenario.no.longer.exists..The.village.children.disappeared.out.of.thin.air..adults.panicked..the.village.s.one.eyed.old.yin.and.yang.mother.in.law.said.that.the.legendary.Tiger.Gospel.again.appeared.sixty.years.later..As.the.case.is.unusual..the.police.sent.out.excellent.young.police.officers. <dbl> …
## $ overview_The.sheriff.and.his.deputies.from.the.first.movie.decide.to.take.a.vacation.in.the.Caribbean..Their.holiday.will.be.short.lived..however..as.the.thawed.murderer.gets.inadvertently.re.frozen.and.brought.back.to.life..As.if.that.weren.t.bad.enough..he.now.has.the.ability.to.remain.frozen.even.in.tropical.temperatures..and.he.s.headed.south.to.settle.some.old.scores. <dbl> …
## $ overview_The.year.is.1955..NATO.and.the.Allied.Forces.have.been.conducting.secret..occult.experiments.in.a.bid.to.win.the.Arms.Race..Now..they.have.finally.succeeded.but.what.the.Army.has.unleashed.threatens.to.tear.our.world.apart..One.woman.must.lead.the.only.survivors.past.horrors.that.the.military.has.no.way.to.control...and.fight.to.close.what.should.never.have.been.opened. <dbl> …
## $ overview_Three.not.so.cool.school.friends.decide.to.try.a.old.voodoo.ritual..Later..they.die.in.a.car.accident..but.live.on.as.zombies..But.being.a.zombie.has.advantages..too... <dbl> …
## $ overview_Three.vixens.have.meditated.for.1.000.years.to.able.to.shed.their.animal.natures.and.become.human..For.the.final.month.of.their.rigors..they.have.moved.near.a.village.where.women.pray.to.a.god.of.fertility..One.sister.visits.the.god.s.temple.and.thinks.lustful.thoughts..As.she.leaves..a.priest.confronts.her..warning.of.dire.consequences.and.of.demons.that.will.try.to.stop.the.vixens..transformation..Soon..the.youngest.sister.saves.a.poor.scholar.from.bandits.and.becomes.enamored.of.him..Each.sister.visits.him..and.before.long..the.youth.has.made.love.to.all.three..After.they.invite.him.to.stay.with.them..the.playfulness.takes.a.scary.turn..Where.can.they.turn.for.help. <dbl> …
## $ overview_Todd.Truman..a.struggling.writer..is.engaged.to.be.married.to.his.beautiful.girlfriend..Jennifer.Davis..When.Jennifer.starts.having.nightmares.due.to.the.stress.of.the.wedding..Todd.seizes.his.window.of.opportunity..He.starts.to.capitalize.on.her.misery.by.turning.these.nightmares.into.his.first.great.novel..When.the.manuscript.catches.the.eye.of.a.local.literary.agent..Todd.s.dreams.of.success.are.finally.realized..Jennifer.begins.to.take.drugs.to.stop.the.nightmares.and.her.daily.suffering..How.far.is.Todd.willing.to.go.to.ensure.his.success. <dbl> …
## $ overview_TOXIC.LULLABY.tells.the.story.of.Eloise..who.wakes.up.in.a.destroyed..life.threatening.world.after.a.bad.drug.trip..She.joins.a.group.of.people..who.like.her..are.motivated.by.the.longing.to.flee.this.nightmare..but.it.s.hard.to.tell.who.ll.carry.the.day..the.survivors.or.the.zombies. <dbl> …
## $ overview_Two.young.journalists.fight.against.an.evil.spirit. <dbl> …
## $ overview_When.a.career.conjure.woman.brings.home.an.evil.spirit..she.and.her.roommate.must.divide.banishment.duties.with.no.time.to.consult.the.chore.wheel. <dbl> …
## $ overview_When.an.ex.con.takes.a.job.as.a.handyman.for.an.unstable.elderly.woman.to.avoid.a.parole.violation..it.becomes.a.choice.he.may.regret. <dbl> …
## $ overview_When.Ann..husband.George..and.son.Georgie.arrive.at.their.holiday.home.they.are.visited.by.a.pair.of.polite.and.seemingly.pleasant.young.men..Armed.with.deceptively.sweet.smiles.and.some.golf.clubs..they.proceed.to.terrorize.and.torture.the.tight.knit.clan..giving.them.until.the.next.day.to.survive. <dbl> …
## $ overview_When.two.micro.brewers.screw.over.the.town.witch..she.curses.their.latest.batch.of.beer..From.that.point.on..everyone.who.drinks.it.suffers.a.gruesome.ironic.death...WITCH.S.BREW.is.the.fourth.feature.film.from.Midnight.Crew.Studios. <dbl> …
## $ overview_You.wouldn.t.wanna.be.stuck.at.work.when.the.Zombie.Apocalypse.breaks.out..Especially.if.you.work.at.a.housing.for.the.mentally.challenged. <dbl> …
## $ overview_Young.Christopher.has.just.enrolled.at.the.prestigious.Carmichael.Bible.College..managed.by.the.somewhat.unusual.Mrs..Bouvier..After.some.unexplained.disappearances..Christopher.does.some.exploring.and.discovers.that.Mrs..Bouvier.and.the.Reverend.Carmichael.have.some.very.unwholesome.intentions.for.the.young.men.of.their.school..Will.Christopher.graduate.with.his.body.and.soul.intact. <dbl> …Evaluate Models
tune::show_best(xgboost_tune, metric = "rmse")## # A tibble: 5 × 12
## trees min_n tree_depth learn_rate loss_reduction sample_size .metric
## <int> <int> <int> <dbl> <dbl> <dbl> <chr>
## 1 1309 24 2 0.00694 2.65e+ 0 0.905 rmse
## 2 1020 11 5 0.00253 4.65e- 5 0.672 rmse
## 3 431 9 10 0.0100 7.91e-10 0.427 rmse
## 4 1926 32 7 0.0768 5.00e- 8 0.553 rmse
## 5 121 34 15 0.212 1.08e- 1 0.110 rmse
## # ℹ 5 more variables: .estimator <chr>, mean <dbl>, n <int>, std_err <dbl>,
## # .config <chr>xgboost_fw <- tune::finalize_workflow(xgboost_workflow,
tune::select_best(xgboost_tune, metric = "rmse"))
data_fit <- tune::last_fit(xgboost_fw, data_split)tune::collect_metrics(data_fit)## # A tibble: 2 × 4
## .metric .estimator .estimate .config
## <chr> <chr> <dbl> <chr>
## 1 rmse standard 0.928 Preprocessor1_Model1
## 2 rsq standard 0.0632 Preprocessor1_Model1tune::collect_predictions(data_fit) %>%
ggplot(aes(vote_average, .pred)) +
geom_point(alpha = 0.3, fill= "midnightblue") +
geom_abline(lty = 2, color = "gray50") +
coord_fixed()
Experimenting with different preprocessing techniques
set.seed(123)
data_split <- initial_split(data, strata = vote_average)
data_train <- training(data_split)
data_test <- testing(data_split)
set.seed(234)
data_folds <- vfold_cv(data_train, strata = vote_average)## Warning: The number of observations in each quantile is below the recommended threshold of 20.
## • Stratification will use 3 breaks instead.# Setting up our preprocessing using step_tf
movie_rec <-
recipe(vote_average ~ overview, data = data_train) %>%
step_tokenize(overview) %>%
step_tokenfilter(overview, max_tokens = 100) %>%
step_tf(overview) # Attempting to use the SVM Model specification
library(parsnip)
svm_spec <-
svm_linear( mode = "regression") %>%
set_engine("kernlab")# Creating the workflow
movie_workflow_svm <-
workflow() %>%
add_recipe(movie_rec) %>%
add_model(svm_spec)# Fit workflow to the training data
library(kernlab)## Warning: package 'kernlab' was built under R version 4.4.1set.seed(345)
movie_fit_svm <-
movie_workflow_svm %>%
fit(data = data_train)## Setting default kernel parameters# Evaluating the model
library(yardstick)
movie_predictions_svm <- predict(movie_fit_svm, data_test) %>%
bind_cols(data_test)
# Evaluting performance
movie_metrics_svm <- movie_predictions_svm %>%
metrics(truth = vote_average, estimate = .pred)
movie_metrics_svm## # A tibble: 3 × 3
## .metric .estimator .estimate
## <chr> <chr> <dbl>
## 1 rmse standard 1.42
## 2 rsq standard 0.0174
## 3 mae standard 1.11Summary
In this analysis I had initially used a basic linear regression model and added improvements throughout to enhace the predictive power of that model. I specifically used a text preprocessing pipeline with step_tokenize, step_tokenfilter, and step_tf to process the “overview” text data. I then transformed the text into term frequency values, so I could capture relevant info from the descriptions of the movies themselves, which might have contained patterns helpful for predicting vote_aveage, which was my target variable.
I then proceeded to replace my basic linear regression model with the SVM linear model, which is better at handling complex relationships within data. After training and evaluating both models I have observed that my changes have made an impact on the Root Mean Squared Error. The RMSE has improved showing that the SVM model might have been better at minimizing prediction errors compared to the basic linear regression model. This new model showed more variance in vote_average than the basic linear regression model I first used.