import numpy as np

class environment:
    def __init__(self):
        self.envData = -1*np.ones( (5,4) ) #*\label{code:robocliff:1}
        self.envData[1,3] = self.envData[2,3] = self.envData[3,3] = -1000 #*\label{code:robocliff:2}
        self.envData[4,3] = 1000 #*\label{code:robocliff:3}
        self.dogX = 0; self.dogY = 3 #*\label{code:robocliff:4}
        
    def setDog(self, x=0, y=3):
        self.dogX = x; self.dogY = y #*\label{code:robocliff:8}
        
    def move(self,a):  
        if a == 0 : self.dogY -= 1 # a = 0 east
        if a == 1 : self.dogX += 1 # a = 1 south 
        if a == 2 : self.dogY += 1 # a = 2 west
        if a == 3 : self.dogX -= 1 # a = 3 north
        if self.dogY >3 or self.dogY <0 or self.dogX >4 or self.dogX <0: #*\label{code:robocliff:5}
            r = -10
        else: 
            r = self.envData[self.dogX,self.dogY]
        if self.dogY <0: self.dogY = 0 #*\label{code:robocliff:6}
        if self.dogX <0: self.dogX = 0
        if self.dogX >4: self.dogX = 4
        if self.dogY >3: self.dogY = 3 #*\label{code:robocliff:7}
        return(r)
        
    def sensors(self):
        return(self.dogX,self.dogY) #*\label{code:robocliff:8} 
        
class learningDog:
    def __init__(self, env,vareps):
        self.hatQ = np.zeros( (5,4,4) ) #*\label{code:robocliff:9} 
        self.env = env
        self.vareps = vareps
        
    def _chooseAction(self,x,y):
        RonnieSoak = np.random.rand(1)
        if RonnieSoak < self.vareps:
            a = np.random.randint(4)
        else:
            localQ = self.hatQ[x,y,:]
            a = np.argmax(localQ)
        return(a)
        
    def learn(self, gamma, trainCycles):
        for i in range(trainCycles):
            (x,y) = self.env.sensors()
            a = self._chooseAction(x,y)
            r = self.env.move(a)
            (xN,yN) = self.env.sensors()
            localQ = self.hatQ[xN,yN,:]
            self.hatQ[x,y,a] = r + gamma * np.max(localQ) #*\label{code:robocliff:53} 
            if r == 1000: self.env.setDog()  #*\label{code:robocliff:10} 
            
    def show(self, x=0, y=3):
        self.env.setDog(x,y)
        r = -1
        counter = 0 
        while r != 1000 and counter < 100:
            counter += 1
            (x,y) = self.env.sensors()
            a = self._chooseAction(x,y)
            r = self.env.move(a)
            print(a,r)

if __name__ == '__main__':
    np.random.seed(42)
    myenv = environment()
    robohund = learningDog(myenv,0.1)
    robohund.learn(0.4,2000)
    print('Start bei 0,3')
    robohund.show()
    print('Start bei 3,0')
    robohund.show(x=3,y=0)
                
                
    