### Closed-form solution of minimum of parabola

f <- function(x) {
  1.2 * (x-2)^2 + 3.2
}

grad <- function(x) {
  1.2 * 2 * (x-2)
}

secondGrad <- function(x) {
  2.4
}

xs <- seq(0,4,len=20)
origpar <- par(mfrow=c(2,2),mar=c(5,5,1,1))
plot(xs,f(xs),type="l",xlab="x",ylab=expression(1.2(x-2)^2 +3.2))
### df/dx = 2.4(x-2)
### df/dx = 0 --->  0 = 2.4x - 4.8 --->  x = 2
lines(c(2,2),c(3,8),col="red",lty=2)
text(2.1,7,"Closed-form solution",col="red",pos=4)
### gradient descent
x <- 0.1
xtrace <- x
ftrace <- f(x)
stepFactor <- 0.6 ### try larger and smaller values (0.8 and 0.01)
for (step in 1:100) {
  x <- x - stepFactor * grad(x)
  xtrace <- c(xtrace,x)
  ftrace <- c(ftrace,f(x))
}
lines(xtrace,ftrace,type="b",col="blue")
text(0.5,6,"Gradient Descent",col="blue",pos=4)

### Now do the same using gradientDescents.R

source("gradientDescents.R")

x <- 0.1
result <- steepest(x, f, grad, stepsize=0.6, nIterations=100, xtracep=TRUE, ftracep=TRUE)

plot(xs,f(xs),type="l",xlab="x",ylab=expression(1.2(x-2)^2 +3.2))
lines(result$xtrace,result$ftrace,type="b",col="blue")
text(0.5,6,"Gradient Descent with steepest()",col="blue",pos=4)

### Now, with Newton's method

plot(xs,f(xs),type="l",xlab="x",ylab=expression(1.2(x-2)^2 +3.2))

x <- 0.1
xtrace <- x
ftrace <- f(x)
for (step in 1:100) {
  x <- x - grad(x)/secondGrad(x)
  xtrace <- c(xtrace,x)
  ftrace <- c(ftrace,f(x))
}

lines(xtrace,ftrace,type="b",col="blue")
text(0.5,6,"Newton's Gradient Descent",col="blue",pos=4)


### Again, with Scaled Conjugate Gradient
x <- 0.1
result <- scg(x, f, grad, nIterations=100, xtracep=TRUE, ftracep=TRUE)

plot(xs,f(xs),type="l",xlab="x",ylab=expression(1.2(x-2)^2 +3.2))
lines(result$xtrace,result$ftrace,type="b",col="blue")
text(0.5,6,"Gradient Descent with scg()",col="blue",pos=4)

par(origpar)